added offline tile storage mgmt library & demo app

This commit is contained in:
andygup 2013-10-07 11:03:58 -06:00
parent 957f3bf46d
commit 8f59cd82cf
7 changed files with 768 additions and 5 deletions

View File

@ -1,10 +1,18 @@
offline-editor-js
=================
Experimental JavaScript library that auto-detects an offline condition and stores FeatureLayer edit activities until a connection is reestablished. No longer will offline edit be the sole domain of native SDKs!
Experimental JavaScript library that auto-detects an offline condition and stores FeatureLayer edit activities until a connection is reestablished. Works with adds, updates and deletes.
Includes several libraries:
- OfflineStore - overrides applyEdits() method
- OfflineTileStore - stores tiles for offline pan and zoom.
- OfflineFeatureStore - **TBD** (manages features for offline usage)
##How to use?
The easiest approach is to simply use the library to override applyEdits():
**Step 1.** The library provides a constructor that can simply be used in place of the traditional applyEdit() method. It does all the rest of the work for you:
var offlineStore = new OfflineStore(map);
@ -20,25 +28,27 @@ While the library works in Chrome, Firefox and Safari with the internet turned o
##Features
* Override the applyEdits() method.
* Can store base map tiles for offline pan and zoom.
* Automatic offline/online detection. Once an offline condition exists the library starts storing the edits. And, as soon as it reconnects it will submit the updates.
* Can store dozens or hundreds of edits.
* Currently works with Points, Polylines and Polygons.
* Indexes edits for successful/unsuccessful update validation as well as for more advanced workflows.
* Monitors available storage and is configured by default to stop edits at a maximum threshold and alert that the threshold has been reached. This is intended to help prevent data loss.
##API
##OfflineStore Library
####OfflineStore(/\* Map \*/ map)
* Constructor. Requires a reference to an ArcGIS API for JavaScript Map.
####applyEdits(/\* Graphic \*/ graphic,/\* FeatureLayer \*/ layer, /\* String \*/ enumValue)
* Method.
* Method. Overrides FeatureLayer.applyEdits().
####getStore()
* Returns an array of Graphics.
* Returns an array of Graphics from localStorage.
####getLocalStoreIndex()
* Returns the index as an array of JSON objects. The objects are constructor like this:
* Returns the index as an array of JSON objects. An internal index is used to keep track of adds, deletes and updates. The objects are constructed like this:
{"id": object610,"type":"add","success":"true"}
@ -68,7 +78,21 @@ While the library works in Chrome, Firefox and Safari with the internet turned o
}
##OfflineTileStore Library
####OfflineTileStore()
* Constructor. Stores tiles for offline panning and zoom.
####storeLayer()
* Stores tiled in either localStorage or IndexedDB if it is available. Storage process is initiated by forcing a refresh on the basemap layer.
####useIndexedDB
* Property. Manually sets whether library used localStorage or IndexedDB. Default is false.
####getLocalStorageUsed()
* Returns amount of storage used by the calling domain. Typical browser limit is 5MBs.
##Testing
Run Jasmine's SpecRunner.html in a browser. You can find it in the /test directory.

89
tiles/index.html Normal file
View File

@ -0,0 +1,89 @@
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=7, IE=9, IE=10">
<meta name="viewport" content="initial-scale=1, maximum-scale=1,user-scalable=no">
<title>Store images in localStorage</title>
<link rel="stylesheet" href="http://js.arcgis.com/3.7/js/dojo/dijit/themes/claro/claro.css">
<link rel="stylesheet" href="http://js.arcgis.com/3.7/js/esri/css/esri.css">
<style>
html, body { height: 100%; width: 100%; margin: 0; padding: 0; }
#map{
padding:0;
}
</style>
<script src="./src/OfflineTileStore.js"></script>
<!--<script src="./src/dbStore.js"></script>-->
<script src="./src/dbStore2.js"></script>
<script>var dojoConfig = {parseOnLoad: true};</script>
<script src="http://js.arcgis.com/3.7/"></script>
<script>
dojo.require("esri.map");
dojo.require("dojox.encoding.digests._base");
dojo.require("dijit.layout.BorderContainer");
dojo.require("dijit.layout.ContentPane");
var map;
var database;
var dbSupport = false;
var offlineTileStore = null;
require([
"esri/map"
],function(Map){
database = new dbStore();
dbSupport = database.isSupported();
if(dbSupport == true){
database.init(function(evt,err){
console.log("dbstore init: " + evt);
}.bind(this));
}
map = new esri.Map("map", {
basemap: "streets",
center: [2.352, 48.87],
zoom: 12
});
map.on("layer-add-result", function(evt){
try{
offlineTileStore = new OfflineTileStore(map)
console.log("Local storage used: " + offlineTileStore.getlocalStorageUsed())
}
catch(err){
console.log("err " + err.stack)
};
}.bind(this));
})
function deleteStorage(){
localStorage.clear();
}
function getSize(){
//database.testGet();
database.size(function(evt,err){
console.log("GET " + evt + " MBs, err: " + err);
})
}
function testRefresh(){
offlineTileStore.storeLayer();
}
</script>
</head>
<body class="claro">
<button onclick="deleteStorage()">Delete localStore</button>
<button onclick="getSize()">Get db size</button>
<button onclick="testRefresh()">Refresh</button>
<div id="map" data-dojo-type="dijit.layout.ContentPane" data-dojo-props="region:'center'" style="overflow:hidden;">
</div>
</body>
</html>

137
tiles/proxy.php Executable file
View File

@ -0,0 +1,137 @@
<?php
/***************************************************************************
* USAGE
* [1] http://<this-proxy-url>?<arcgis-service-url>
* [2] http://<this-proxy-url>?<arcgis-service-url> (with POST body)
* [3] http://<this-proxy-url>?<arcgis-service-url>?token=ABCDEFGH
*
* note: [3] is used when fetching tiles from a secured service and the
* JavaScript app sends the token instead of being set in this proxy
*
* REQUIREMENTS
* - cURL extension for PHP must be installed and loaded. To load it,
* add the following lines to your php.ini file:
* extension_dir = "<your-php-install-location>/ext"
* extension = php_curl.dll
*
* - Turn OFF magic quotes for incoming GET/POST data: add/modify the
* following line to your php.ini file:
* magic_quotes_gpc = Off
*
***************************************************************************/
/***************************************************************************
* <true> to only proxy to the sites listed in '$serverUrls'
* <false> to proxy to any site (are you sure you want to do this?)
*/
$mustMatch = true;
/***************************************************************************
* ArcGIS Server services this proxy will forward requests to
*
* 'url' = location of the ArcGIS Server, either specific URL or stem
* 'matchAll' = <true> to forward any request beginning with the URL
* <false> to forward only the request that exactly matches the url
* 'token' = token to include for secured service, if any, otherwise leave it
* empty
*/
$serverUrls = array(
array( 'url' => 'http://sampleserver1.arcgisonline.com/ArcGIS/rest/services/', 'matchAll' => true, 'token' => '' ),
array( 'url' => 'http://services.arcgisonline.com/ArcGIS/rest/services/', 'matchAll' => true, 'token' => '' ),
array( 'url' => 'http://sampleserver2.arcgisonline.com/ArcGIS/rest/services/', 'matchAll' => true, 'token' => '' ),
array( 'url' => 'http://sampleserver1a.arcgisonline.com/arcgisoutput/', 'matchAll' => true, 'token' => '' ),
array( 'url' => 'http://sampleserver1b.arcgisonline.com/arcgisoutput/', 'matchAll' => true, 'token' => '' ),
array( 'url' => 'http://sampleserver1c.arcgisonline.com/arcgisoutput/', 'matchAll' => true, 'token' => '' )
);
/***************************************************************************/
function is_url_allowed($allowedServers, $url) {
$isOk = false;
$url = trim($url, "\/");
for ($i = 0, $len = count($allowedServers); $i < $len; $i++) {
$value = $allowedServers[$i];
$allowedUrl = trim($value['url'], "\/");
if ($value['matchAll']) {
if (stripos($url, $allowedUrl) === 0) {
$isOk = $i; // array index that matched
break;
}
}
else {
if ((strcasecmp($url, $allowedUrl) == 0)) {
$isOk = $i; // array index that matched
break;
}
}
}
return $isOk;
}
// check if the curl extension is loaded
if (!extension_loaded("curl")) {
header('Status: 500', true, 500);
echo 'cURL extension for PHP is not loaded! <br/> Add the following lines to your php.ini file: <br/> extension_dir = &quot;&lt;your-php-install-location&gt;/ext&quot; <br/> extension = php_curl.dll';
return;
}
$targetUrl = $_SERVER['QUERY_STRING'];
if (!$targetUrl) {
header('Status: 400', true, 400); // Bad Request
echo 'Target URL is not specified! <br/> Usage: <br/> http://&lt;this-proxy-url&gt;?&lt;target-url&gt;';
return;
}
$parts = preg_split("/\?/", $targetUrl);
$targetPath = $parts[0];
// check if the request URL matches any of the allowed URLs
if ($mustMatch) {
$pos = is_url_allowed($serverUrls, $targetPath);
if ($pos === false) {
header('Status: 403', true, 403); // Forbidden
echo 'Target URL is not allowed! <br/> Consult the documentation for this proxy to add the target URL to its Whitelist.';
return;
}
}
// add token (if any) to the url
$token = $serverUrls[$pos]['token'];
if ($token) {
$targetUrl .= (stripos($targetUrl, "?") !== false ? '&' : '?').'token='.$token;
}
// open the curl session
$session = curl_init();
// set the appropriate options for this request
$options = array(
CURLOPT_URL => $targetUrl,
CURLOPT_HEADER => false,
CURLOPT_HTTPHEADER => array(
'Content-Type: ' . $_SERVER['CONTENT_TYPE'],
'Referer: ' . $_SERVER['HTTP_REFERER']
),
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true
);
// put the POST data in the request body
$postData = file_get_contents("php://input");
if (strlen($postData) > 0) {
$options[CURLOPT_POST] = true;
$options[CURLOPT_POSTFIELDS] = $postData;
}
curl_setopt_array($session, $options);
// make the call
$response = curl_exec($session);
$code = curl_getinfo($session, CURLINFO_HTTP_CODE);
$type = curl_getinfo($session, CURLINFO_CONTENT_TYPE);
curl_close($session);
// set the proper Content-Type
header("Status: ".$code, true, $code);
header("Content-Type: ".$type);
echo $response;
?>

View File

@ -0,0 +1,203 @@
/**
* Library for handling the storage of map tiles.
* Can use localStorage or IndexedDB. Local storage is supported in
* more browsers (caniuse.com) however it is significantly more
* limited in size.
* NOTE: Uses localStorage by default. Override with useIndexedDB property.
* NOTE: if you use IndexedDB be sure to verify if its available for use.
* @param map
* @constructor
*
* Author: Andy Gup (@agup)
*/
var OfflineTileStore = function(/* Map */ map) {
this.ioWorker = null;
this.extend = null;
this.storage = 0;
this.map = map;
this.dbStore = null //indexedDB
this.useIndexedDB = false;
/**
* Provides control over allow/disallow values to be
* written to storage. Can be used for testing as well.
* @type {boolean}
*/
this.allowCache = true;
/**
* Private Local ENUMs (Constants)
* Contains required configuration info.
* @type {Object}
* @returns {*}
* @private
*/
this._localEnum = (function(){
var values = {
TIMEOUT : 20, /* Seconds to wait for all tile requests to complete */
LOCAL_STORAGE_MAX_LIMIT : 4.75, /* MB */ /* Most browsers offer default storage of ~5MB */
LS_TILE_COUNT : "tile_count",
WORKER_URL : "./src/ioWorker.js" /* child process for gathering tiles */
}
return values;
});
/**
* Determines total storage used for this domain.
* @returns Number MB's
*/
this.getlocalStorageUsed = function(){
var mb = 0;
//IE hack
if(window.localStorage.hasOwnProperty("remainingspace")){
//http://msdn.microsoft.com/en-us/library/ie/cc197016(v=vs.85).aspx
mb = window.localStorage.remainingSpace/1024/1024;
}
else{
for(var x in localStorage){
//Uncomment out console.log to see *all* items in local storage
//console.log(x+"="+((localStorage[x].length * 2)/1024/1024).toFixed(2)+" MB");
mb += localStorage[x].length
}
}
return Math.round(((mb * 2)/1024/1024) * 100)/100;
}
/**
* Refreshes base map and stores tiles.
* If they are already in database they are ignored.
*/
this.storeLayer = function(){
this.tileCount = 0;
this.extendLayer(function(/* boolean */ evt){
var ids = map.layerIds;
var layer = map.getLayer(ids[0]);
layer.refresh();
}.bind(this));
}
this.extendLayer = function(callback){
if(this.extend == null){
var count = 0;
var allow = this.allowCache;
var worker = this.ioWorker;
var db = database;
var indexDB = this._useIndexedDB;
this.extend = dojo.extend(esri.layers.ArcGISTiledMapServiceLayer, { //extend ArcGISTiledMapServiceLayer to use localStorage if available, else use worker to request tile and store in local storage.
getTileUrl : function(level, row, col) {
this.tileCount++;
count++; //count number of tiles
console.log("Count " + count);
localStorage.setItem("tile_count",count);
var url = this._url.path + "/tile/" + level + "/" + row + "/" + col;
if(indexDB == true){
database.get(url,function(event,result){
console.log("img: " + result.img + ", event.url: " + result.url);
if(event == true){
console.log("in indexed db storage");
return "data:image;base64," + result.img;
}
else{
console.log("not in indexed db storage, pass url and load tile", url);
worker.postMessage([url]);
return url;
}
}.bind(this))
}
else{
if(localStorage.getItem(url) !== null) {
console.log("in local storage");
return "data:image;base64," + localStorage.getItem(url);
}
else if(allow == true) {
console.log("not in local storage, pass url and load tile", url);
worker.postMessage([url]);
return url;
}
}
}});
callback(true);
}
else{
callback(false);
}
}
/**
* Load src
* TO-DO: Needs to be made AMD compliant!
* @param urlArray
* @param callback
* @private
*/
this._loadScripts = function(/* Array */ urlArray, callback)
{
count = 0;
for(var i in urlArray){
try{
var head = document.getElementsByTagName('head')[0];
var script = document.createElement('script');
script.type = 'text/javascript';
script.src = urlArray[i];
script.onreadystatechange = function(){
count++;
console.log("Script loaded. " + this.src);
if(count == urlArray.length) callback();
};
script.onload = function(){
count++;
console.log("Script loaded. " + this.src);
if(count == urlArray.length) callback();
};
head.appendChild(script);
}
catch(err){
console.log("_loadScripts: " + err.stack);
}
}
}
this.initLocalStorage = function() {
var tempArray = [];
var tempCount = 0;
this.dbStore = new dbStore();
this.ioWorker = new Worker(this._localEnum().WORKER_URL);
this.ioWorker.onmessage = function(evt) {
this.storage = this.getlocalStorageUsed();
console.log("Worker to Parent: ", evt.data[0]);
console.log("localStorage used: " + this.getlocalStorageUsed());
try {
localStorage.setItem(evt.data[0], evt.data[1]);
tempCount++;
tempArray.push({url:evt.data[0],img: evt.data[1]});
} catch(error) {
console.log('Problem adding tile to local storage. Storage might be full');
}
var count = parseFloat(localStorage.getItem(this._localEnum().LS_TILE_COUNT));
if(tempCount == count){
localStorage.setItem(this._localEnum().LS_TILE_COUNT,0);
database.add(tempArray,function(evt,err){
evt == true ? console.log("Done") : console.log("init " + err);
});
}
}.bind(this);
}
this._init = function(){
this.initLocalStorage();
}.bind(this)()
}

73
tiles/src/_base.js Normal file
View File

@ -0,0 +1,73 @@
var Base64Utils = (function(){
var d={};
d.outputTypes={
// summary:
// Enumeration for input and output encodings.
Base64:0, Hex:1, String:2, Raw:3
};
// word-based addition
d.addWords=function(/* word */a, /* word */b){
// summary:
// add a pair of words together with rollover
var l=(a&0xFFFF)+(b&0xFFFF);
var m=(a>>16)+(b>>16)+(l>>16);
return (m<<16)|(l&0xFFFF); // word
};
// word-based conversion method, for efficiency sake;
// most digests operate on words, and this should be faster
// than the encoding version (which works on bytes).
var chrsz=8; // 16 for Unicode
var mask=(1<<chrsz)-1;
d.stringToWord=function(/* string */s){
// summary:
// convert a string to a word array
var wa=[];
for(var i=0, l=s.length*chrsz; i<l; i+=chrsz){
wa[i>>5]|=(s.charCodeAt(i/chrsz)&mask)<<(i%32);
}
return wa; // word[]
};
d.wordToString=function(/* word[] */wa){
// summary:
// convert an array of words to a string
var s=[];
for(var i=0, l=wa.length*32; i<l; i+=chrsz){
s.push(String.fromCharCode((wa[i>>5]>>>(i%32))&mask));
}
return s.join(""); // string
}
d.wordToHex=function(/* word[] */wa){
// summary:
// convert an array of words to a hex tab
var h="0123456789abcdef", s=[];
for(var i=0, l=wa.length*4; i<l; i++){
s.push(h.charAt((wa[i>>2]>>((i%4)*8+4))&0xF)+h.charAt((wa[i>>2]>>((i%4)*8))&0xF));
}
return s.join(""); // string
}
d.wordToBase64=function(/* word[] */wa){
// summary:
// convert an array of words to base64 encoding, should be more efficient
// than using dojox.encoding.base64
var p="=", tab="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", s=[];
for(var i=0, l=wa.length*4; i<l; i+=3){
var t=(((wa[i>>2]>>8*(i%4))&0xFF)<<16)|(((wa[i+1>>2]>>8*((i+1)%4))&0xFF)<<8)|((wa[i+2>>2]>>8*((i+2)%4))&0xFF);
for(var j=0; j<4; j++){
if(i*8+j*6>wa.length*32){
s.push(p);
} else {
s.push(tab.charAt((t>>6*(3-j))&0x3F));
}
}
}
return s.join(""); // string
};
return d;
})();

205
tiles/src/dbStore2.js Normal file
View File

@ -0,0 +1,205 @@
/**
* Library for handling the storing of map tiles in IndexedDB.
*
* Author: Andy Gup (@agup)
*/
var dbStore = function(){
/**
* Internal reference to the local database
* @type {null}
* @private
*/
this._db = null;
/**
* Private Local ENUMs (Constants)
* Contains required configuration info.
* @type {Object}
* @returns {*}
* @private
*/
this._localEnum = (function(){
var values = {
DB_NAME : "offline_tile_store" /* Seconds to wait for all tile requests to complete */
}
return values;
});
/**
* Determines if indexedDB is supported
* @returns {boolean}
*/
this.isSupported = function(){
window.indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB;
if(!window.indexedDB){
return false;
}
return true;
}
/**
* Adds an object to the database
* @param urlData
* @param callback callback(boolean, err)
*/
this.add = function(/* Array */ urlData,callback){
try{
var transaction = this._db.transaction(["tilepath"],"readwrite");
transaction.oncomplete = function(event) {
callback(true);
};
transaction.onerror = function(event) {
callback(false,event.target.error.message)
};
var objectStore = transaction.objectStore("tilepath");
for (var i in urlData) {
var request = objectStore.add(urlData[i]);
request.onsuccess = function(event) {
// event.target.result == customerData[i].ssn;
console.log("item added to db " + event.target.result);
};
}
}
catch(err){
console.log("dbstore: " + err.stack);
callback(false,err.stack);
}
}
/**
* Retrieve a record.
* @param url
* @param callback
*/
this.get = function(/* String */ url,callback){
if(this._db != null){
var index = this._db.transaction(["tilepath"]).objectStore("tilepath").index("url");
index.get(url).onsuccess = function(event){
var result = event.target.result;
if(result == null){
callback(false,"not found");
}
else{
callback(true,result);
}
}
}
}
/**
* Deletes entire database
* @param callback callback(boolean, err)
*/
this.deleteAll = function(callback){
if(this._db != null){
var transaction = this._db.transaction(["tilepath"],"readwrite").objectStore("tilepath");
transaction.clear();
transaction.onsuccess = function(event){
callback(true);
}
transaction.onerror = function(err){
callback(false,err);
}
}
else{
callback(false,null);
}
}
/**
* Delete an individual entry
* @param url
* @param callback callback(boolean, err)
*/
this.delete = function(/* String */ url,callback){
if(this._db != null){
var transaction = this._db.transaction(["tilepath"],"readwrite")
.objectStore("tilepath")
.delete(url);
transaction.onsuccess = function(event){
callback(true);
}
transaction.onerror = function(err){
callback(false,err);
}
}
else{
callback(false,null);
}
}
/**
* Provides a rough, approximate size of database in MBs.
* @param callback callback(size, null) or callback(null, error)
*/
this.size = function(callback){
if(this._db != null){
var size = 0;
var transaction = this._db.transaction(["tilepath"])
.objectStore("tilepath")
.openCursor();
transaction.onsuccess = function(event){
var cursor = event.target.result;
if(cursor){
var url = cursor.value;
var json = JSON.stringify(url);
size += this.stringBytes(json);
cursor.continue();
}
else{
size = Math.round(((size * 2)/1024/1024) * 100)/100;
callback(size,null);
}
}.bind(this);
transaction.onerror = function(err){
callback(null,err);
}
}
else{
callback(null,null);
}
}
this.stringBytes = function(str) {
var b = str.match(/[^\x00-\xff]/g);
return (str.length + (!b ? 0: b.length));
}
this.init = function(callback){
var request = indexedDB.open(this._localEnum().DB_NAME, 2);
request.onerror = function(event) {
console.log("indexedDB error: " + event.target.errorCode);
callback(false,event.target.errorCode);
};
request.onupgradeneeded = (function(event) {
var db = event.target.result;
// Create an objectStore to hold information about our map tiles.
var objectStore = db.createObjectStore("tilepath", {
autoIncrement: true
});
// Create an index to search urls. We may have duplicates
// so we can't use a unique index.
objectStore.createIndex("url", "url", { unique: false });
}.bind(this))
request.onsuccess = (function(event){
this._db = event.target.result;
console.log("database opened successfully");
callback(true);
}.bind(this))
}
}

32
tiles/src/ioWorker.js Normal file
View File

@ -0,0 +1,32 @@
// Base64 conversion functions
importScripts("_base.js");
// Parent to worker
onmessage = function(evt) {
getImages(evt.data);
};
function getImages(urls) {
for (var i = 0; i < urls.length; i++) {
var imgBytes = getImage(urls[i]);
if (imgBytes) {
var encoded = Base64Utils.wordToBase64(Base64Utils.stringToWord(imgBytes));
postMessage([ urls[i], encoded ]);
}
} // loop
}
function getImage(url) {
url = "../proxy.php?" + url;
var req = new XMLHttpRequest();
req.open("GET", url, false);
req.overrideMimeType("text/plain; charset=x-user-defined");
req.send(null);
if (req.status != 200) {
return "";
}
return req.responseText;
}