geojs/scripts/datastore.js

var crypto = require('crypto');
var fs = require('fs');
var https = require('https');
var path = require('path');
var tar = require('tar');

var registry = {
  'AdderallCities2015.csv': 'c3e984482cc6db1193a6dca2a55396a2baad8541a5c8c679f33750b76f766f40a119ec3e63abbabcd095d752c3df8ce21bf24cbe629502121f24ba90b4b0674c',
  'base-images.tgz': 'cd36e948a639daf4f684343270614902047954a1add9a1845dc426a3d422920e2c008117a07fe9a79be82539891f3c666e4782111cb835b82d9903d53a312441',
  'blue.jpg': '867b1f3c568289efc7d0dba97d827a2bc4d83a7465cebcb3b5aec7bac6a38cf70d037d1814402bc97ad1f2f6737cfb5ce97db0a4fb53a716e77fd3ba57a7ab3b',
  'cities.csv': '5a665e5feda24f28e5cf4ed0801b67e73bbcf3ea781b2e50d11284214e67b25b68e6a1c48da46e5e4d4d0c54c2ec18f88d292224b4541fb279396cf7b94beac9',
  'earthquakes.json': 'f098b6437411384b552419b4a36264c1bb3fed816ccfe9545145175e0b92a0b7ad5ebdcb9dddd0a12a90499143ffa471c02f6e049be5b973db607ff066892500',
  'earthquakes-video.webm': '834a9d05f5fb00145e529fa4b217398db0df188c69d751f3869128b6e9c92d3000f85378752c56d9d9b5fa0870437dd9bdfeb5d62f6c87c2c03a7f1a20ee8523',
  'grid.jpg': '60d201a14c7d31e7881301e6784e0372ddf27f26e5e4eafba9ba39158dfd050e3683faaa660fcde47e6c994dd3ee64c5a5231474ca75090053ef9207fedd9029',
  'hurricanes.json': '0c44cca72c76ead45a2d8084acb3597db4827a880637e5957f1d986f7c2b57986d2bada824756a87e1de730e4078af0a0b6462539bb9d4f764271596a4965773',
  'land_polygons.json': '30a828392d58678599130e0dca6d7f27e7e07e4e5b5d7f7a37871eb395d53d97b76134c0a07e805fbdfac0f42e6d3ca6e287c9727815a9dc711d541b1c8f68a1',
  'land_shallow_topo_2048.png': '8a8330dba5bacdb511038ad0f6ee5a764a40aa7a8868a445749f653ae5d85d8317684ac706e7a9f049590170df6bc3fefc2912d52124d1b3b17aa43c529ff2a8',
  'noaa_prcp.json': 'a03dd083292c6eac86f73dc4512e5f39b7c195402db960a85824d43a144c0af6f1f4ce86c0bb0b8d9c58e19eef6756d610c8c8e2e30d91989012e41457bb59dd',
  'oahu-dense.json': '692a44ec4a18b16d1530403a9a2bf286ff2ef7d45fa58f555c278f91a0fa708b5626a3f38955e06a11cdc06a1009e859328687c1f32a2169e0c8ef1b518418c1',
  'oahu.json': 'e44282c44fa95f0b40c2135ee94ff40a755771f3b4bf9acda0eef0048cd0fb29b3a71352e2eecdca6cfc35fafde96ffb1a9658c9fee2346071808a9123c26cb5',
  'oahu-medium.json': '83375c5c2678b11d8de7f59fd8e2f9b889d3ec20f4a81279d996711bfc0942bd9dcdce149cbca995930459c26b3bf58be60569687085adf361ff805436b75aab',
  'pixelmap_0_0_0.png': '67bc541c8f26de5b644d756bc517b31ef4187c1c609da5291fc446fa7e2ddcf40c75b93cdb6f2e579be596ac1215c5af802c9555474723d3b10713851fbb58e3',
  'pixelmap_1_0_0.png': '41ca6c789bd410b0ea4dd814c92eb6047c098fcc6a8f77c200604e2bff7b7847f6c740e3a411a957a03edc0dfe6ee1383f38c34dceacdfa3e9f6408b7187ee25',
  'pixelmap_1_0_1.png': 'a27aa667e482cd7a7800cbe9c2401b9fbf387bdc67a49b17be04f7aa2846e522aa64383383fb3413d9dc7206e3b90910ccc8d4454ac1b08ebcedc68182c49e99',
  'pixelmap_1_1_0.png': 'b3f020355093ca6fa450550fe9680f549a6dc327a568d439d59e620ee3f9a43b6c88913e5d9344e65bd896cce5494b864051359faaaa4704433d5d0c41da2a43',
  'pixelmap_1_1_1.png': '31060e137ea25029dc977c66af62becf5ab685a3ac254fd8a9953eaa71c9b20efe97cd2d9ba03389e0d5c90c2c6bbe46a43a99c4aef10159b0d3515ee80cf459',
  'red.jpg': '70884f5c3d1747633412dcf64d4886d985b49f508b68eb9f3874b2fdcfad38e7623f4a7c7355baa1656503f46e2e091576a19ded9fad7c7c65387c313206d2e6',
  'roads.json': 'ae8d8b99c3fab73798ddcc246ea53b1bc8d598414d00df4ce697373a343fbf7ffed4dbae5f07a997ac7731b5f86140686b8425b99bd5818b8c9ec68e7c4d3315',
  'sample.json': '5d4b00559f17fac607eafe4a9bb933386ebd572fbe545e6138b8eafbbf510074a8ae398c7df0420a017218af83b18cc322633b77007b63a27dfea7a50db70244',
  'temperature_data.tsv': 'bde5cdad7c4518694393de83da938bcf5d485b41116b51f0fce4b462737992d2973f076c577040fa816873b196c6397244aa03736fc7750a0eea0ba5cbcc48e4',
  'tilefancy.png': '455effa59d421cbb73c6def815813063f1c649363de4007fa0de00cc2e0f24cba745c046266e5a83fc43b121a648017d509d7bf03c30fbee1448817e3f849683',
  'tiles.tgz': 'b777fc6b206a89241250bbfeb17fd2dd36966c7697160c7be71bb5c769db88ae2a652874200f73aac3356bc3e09a878be1daf16e11c4178ddc685f8b4370d55e',
  'white.jpg': 'ea7a9d7ff76775e742572f89e90cce1248ec99c33b2f486e0fa1d19ab461b87dff324533ecb186a3db14e40a3826da97b5d66566360a201228f60140b0e89942'
};

function download(url, dest) {
  return new Promise((resolve, reject) => {
    var file = fs.createWriteStream(dest);
    https.get(url, function (response) {
      response.pipe(file);
      file.on('finish', function () {
        file.close(() => resolve(dest));
      });
    });
  });
}

function checksumFile(hashName, filepath) {
  return new Promise((resolve, reject) => {
    const hash = crypto.createHash(hashName);
    const stream = fs.createReadStream(filepath);
    stream.on('error', err => reject(err));
    stream.on('data', chunk => hash.update(chunk));
    stream.on('end', () => resolve(hash.digest('hex')));
  });
}

function untgz(filepath) {
  var dest = filepath.substr(0, filepath.length - 4);
  fs.mkdirSync(dest, {recursive: true});
  return tar.x({file: filepath, cwd: dest});
}

var base_url = 'https://data.kitware.com/api/v1/file/hashsum/{algo}/{hashvalue}/download';
var algo = 'sha512';

var dest = process.argv[2];

fs.mkdirSync(dest, {recursive: true});
Object.entries(registry).forEach(async ([name, hash]) => {
  var outputPath = path.join(dest, name);
  var url = base_url.replace('{algo}', algo).replace('{hashvalue}', hash);
  var downloaded = false;
  for (var tries = 0; tries < 5; tries += 1) {
    if (tries) {
      console.log(`Downloading ${name}.`);
      await download(url, outputPath);
      downloaded = true;
    }
    if (fs.existsSync(outputPath)) {
      var existingHash = await checksumFile(algo, outputPath);
      if (hash.toLowerCase() === existingHash.toLowerCase()) {
        break;
      }
      if (tries) {
        console.log(`Checksum does not match for ${name}.`);
      }
    }
    downloaded = false;
  }
  if (downloaded && name.endsWith('tgz')) {
    await untgz(outputPath);
  }
});