mirror of
https://github.com/cambecc/air.git
synced 2025-12-08 21:26:22 +00:00
581 lines
19 KiB
JavaScript
581 lines
19 KiB
JavaScript
/**
|
|
* api - a module that defines the behavior of the "air" http server.
|
|
*
|
|
* All files in air/public are mounted at /, and index.html is the primary driver of the visualization.
|
|
* index.html is reused to dynamically generate maps located at:
|
|
*
|
|
* /map/wind/current
|
|
* /map/wind/2013/10/31/15
|
|
* /map/temp/current
|
|
* /map/temp/2013/10/31/15
|
|
* ... and so on
|
|
*
|
|
* The raw data for air samples is made available as JSON at the following paths:
|
|
*
|
|
* /data/wind/current
|
|
* /data/wind/2013/10/31/15
|
|
* /data/temp/current
|
|
* /data/temp/2013/10/31/15
|
|
* ... and so on
|
|
*
|
|
* GETs on data paths cause SQL selects. The results are memoized to avoid repeated db hits.
|
|
*/
|
|
|
|
"use strict";
|
|
|
|
var util = require("util");
|
|
var fs = require("fs");
|
|
var _ = require("underscore");
|
|
var express = require("express");
|
|
var when = require("when");
|
|
var db = require("./db");
|
|
var schema = require("./schema");
|
|
var tool = require("./tool");
|
|
var log = tool.log();
|
|
|
|
var port = process.argv[2];
|
|
var ruleFile = process.argv[5];
|
|
|
|
var validSampleTypes = schema.samples.columns.map(function(column) { return column.name; });
|
|
var validOverlays = _.reject(validSampleTypes, function(s) {
|
|
return s === "date" || s === "stationId" || s === "wv" || s === "wd";
|
|
});
|
|
|
|
// Cache index.html to serve it out. Changes require a restart to pick them up. Need to find a better way to do this.
|
|
var indexHTML = "./public/index.html";
|
|
var indexHTMLText = fs.readFileSync(indexHTML, {encoding: "utf-8"});
|
|
var indexHTMLDate = fs.statSync(indexHTML).mtime;
|
|
|
|
var app = express();
|
|
|
|
app.use(cacheControl());
|
|
app.use(cors());
|
|
app.use(express.compress({filter: compressionFilter}));
|
|
|
|
express.logger.token("date", function() {
|
|
return new Date().toISOString();
|
|
});
|
|
express.logger.token("response-all", function(req, res) {
|
|
return tool.coalesce(res._header, "").trim();
|
|
});
|
|
app.use(express.logger(
|
|
':date - info: :remote-addr :req[cf-connecting-ip] :req[cf-ipcountry] :method :url HTTP/:http-version ' +
|
|
'":user-agent" :referrer :req[cf-ray] :req[accept-encoding]'));
|
|
// '":user-agent" :referrer :req[cf-ray] :req[accept-encoding]\\n:response-all\\n'));
|
|
|
|
//app.get("/about/stations", function(request, response) {
|
|
// var result = {};
|
|
// schema.stations.columns.forEach(function(column) {
|
|
// result[column.name] = column.description;
|
|
// });
|
|
// response.type("json");
|
|
// response.json(prepare(result));
|
|
//});
|
|
|
|
app.get("/data/stations", function(req, res) {
|
|
var stmt = db.selectAll(schema.stations);
|
|
when(db.execute(stmt)).then(
|
|
function(result) {
|
|
// prepareLastModified(res, new Date("????-??-?? ??:??:??Z"));
|
|
res.set("Content-Type", "application/json");
|
|
res.send(JSON.stringify(result.rows));
|
|
},
|
|
handleUnexpected.bind(null, res));
|
|
});
|
|
|
|
//app.get("/stations/geo", function(request, response) {
|
|
// var stmt = db.selectAll(schema.stations);
|
|
// when(db.execute(stmt)).then(
|
|
// function(result) {
|
|
// var out = {
|
|
// type: "FeatureCollection",
|
|
// features: result.rows.map(function(element) {
|
|
// return {
|
|
// type: "Feature",
|
|
// properties: {name: element.id.toString()},
|
|
// geometry: {
|
|
// type: "Point",
|
|
// coordinates: [
|
|
// parseFloat(element.longitude),
|
|
// parseFloat(element.latitude)
|
|
// ]
|
|
// }
|
|
// }
|
|
// })
|
|
// };
|
|
// response.type("json");
|
|
// response.json(prepare(out));
|
|
// },
|
|
// function(error) {
|
|
// response.type("json");
|
|
// response.json(prepare(error.message));
|
|
// });
|
|
//});
|
|
//
|
|
//app.get("/about/samples", function(request, response) {
|
|
// var result = {};
|
|
// schema.samples.columns.forEach(function(column) {
|
|
// result[column.name] = column.description;
|
|
// });
|
|
// response.type("json");
|
|
// response.json(prepare(result));
|
|
//});
|
|
//
|
|
//app.get("/samples/*", function(request, response) {
|
|
// var args = request.params[0].split(/\//);
|
|
// console.log("/samples/* " + util.inspect(args));
|
|
//
|
|
// // sample-type := 'all' | 'temp' | 'hum' | 'wd' | ...
|
|
// // station-id := int
|
|
// // year, month, day, hour := int
|
|
// //
|
|
// // sample-type-path := sample-type [ '/' station-id ]
|
|
// // date-path := year [ '/' month [ '/' day [ '/' hour ] ] ] [ '/' sample-type-path ]
|
|
// // current-path := 'current' [ '/' (date-path | sample-type-path) ]
|
|
// // samples-path := 'samples' [ '/' (current-path | date-path) ]
|
|
// //
|
|
// // examples:
|
|
// // samples/current - all current samples
|
|
// // samples/current/temp - all current temps
|
|
// // samples/current/temp/117 - current temp at station 117
|
|
// // samples/2013/7/temp - all temps for month of 2013-07
|
|
// // samples/2013/7/15/temp - all temps for day of 2013-07-15
|
|
// // samples/2013/7/15/22/temp/117 - temp at 10 PM on 2013-07-15 for station 117
|
|
// // samples/current/-1/temp - all temps at this moment, one year ago today
|
|
// // samples/current/0/0/-7/all/117 - all samples for station 117 at this moment, one week ago
|
|
// //
|
|
// // CONSIDER: significance of date parts determines sample range:
|
|
// // samples/current/0/-1/temp - all temps for last month
|
|
// // samples/current/0/-1/0/temp - all temps for the entire day exactly one month ago
|
|
// // samples/current/0/-1/0/0/temp - all temps for this exact moment one month ago
|
|
//
|
|
// var next;
|
|
// var result = {date: {current: false, parts: [], zone: "+09:00"}, sampleType: null, stationId: null, error: null};
|
|
//
|
|
// function parseSampleTypePath() {
|
|
// result.sampleType = next; // UNDONE: sample type validation -- must be one of no, no2, temp, etc.
|
|
// result.stationId = args.shift(); // UNDONE: stationId validation -- must be numeric
|
|
// return db.selectSamples(schema.samples, schema.stations, result);
|
|
// }
|
|
//
|
|
// function parseDatePath() {
|
|
// do {
|
|
// result.date.parts.push(next * 1); // UNDONE: actual int validation -- must be numeric
|
|
// next = args.shift();
|
|
// } while (_.isFinite(next));
|
|
// return parseSampleTypePath();
|
|
// }
|
|
//
|
|
// function parseCurrentPath() {
|
|
// result.date.current = true; // next == "current";
|
|
// next = args.shift();
|
|
// if (_.isFinite(next)) {
|
|
// return parseDatePath();
|
|
// }
|
|
// return parseSampleTypePath();
|
|
// }
|
|
//
|
|
// function parseSamplesPath() {
|
|
// next = args.shift();
|
|
// if (_.isFinite(next)) {
|
|
// return parseDatePath();
|
|
// }
|
|
// if (next === "current") {
|
|
// return parseCurrentPath();
|
|
// }
|
|
// result.error = "not numeric";
|
|
// }
|
|
//
|
|
// var stmt = parseSamplesPath();
|
|
//
|
|
// if (args.length > 0) {
|
|
// result.error = "too many args";
|
|
// }
|
|
//
|
|
// if (result.error) {
|
|
// response.type("json");
|
|
// return response.json(prepare(result.error));
|
|
// }
|
|
//
|
|
// when(db.execute(stmt)).then(
|
|
// function(result) {
|
|
// response.type("json");
|
|
// response.json(prepare(result.rows));
|
|
// },
|
|
// function(error) {
|
|
// response.type("json");
|
|
// response.json(prepare(error.message));
|
|
// });
|
|
//});
|
|
|
|
function handleUnexpected(res, error) {
|
|
log.error(error.stack);
|
|
res.send(500);
|
|
}
|
|
|
|
/**
|
|
* Returns true if the response should be compressed.
|
|
*/
|
|
function compressionFilter(req, res) {
|
|
return /json|text|javascript|font/.test(res.getHeader('Content-Type'));
|
|
}
|
|
|
|
/**
|
|
* Returns i as an integer if it matches the regex and lies in the range [from, to], otherwise NaN.
|
|
*/
|
|
function parseInt(i, regex, from, to) {
|
|
if (!regex.test(i)) {
|
|
return NaN;
|
|
}
|
|
var result = parseFloat(i);
|
|
if (result !== Math.floor(result) || result < from || to < result) {
|
|
return NaN;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Returns the specified date parts as an array. Any value that is not valid for the date part it represents is
|
|
* parsed as NaN. For example: "2013", "09", "17", "23" yields [2013, 9, 17, 23] whereas "2013", "9.1", "-17", "42"
|
|
* yields [2013, NaN, NaN, NaN].
|
|
*/
|
|
function parseDateParts(year, month, day, hour) {
|
|
return [
|
|
parseInt(year, /^\d{4}$/, 2000, 2100),
|
|
parseInt(month, /^\d{1,2}$/, 1, 12),
|
|
parseInt(day, /^\d{1,2}$/, 1, 31),
|
|
parseInt(hour, /^\d{1,2}$/, 0, 24)
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Casts v to a Number if it is a finite number, otherwise null.
|
|
*/
|
|
function asNullOrNumber(v) {
|
|
return _.isFinite(v) ? +v : null;
|
|
}
|
|
|
|
/**
|
|
* Returns the greater of two dates.
|
|
*/
|
|
function dateMax(a, b) {
|
|
return a < b ? b : a; // Wish I could use Math.max here...
|
|
}
|
|
|
|
function buildResponse(sampleType, rows) {
|
|
// Build JSON response like this, where sampleType is "no2":
|
|
// [
|
|
// {
|
|
// "date": "2013-09-04 16:00:00+09:00",
|
|
// "samples": [{"stationId": "101", "coordinates": [139.768119, 35.692752], "wind": [90, 0.6], "no2": 3}, ...]
|
|
// },
|
|
// ...
|
|
// ]
|
|
|
|
var buckets = {}; // collect rows having common dates into buckets
|
|
rows.rows.forEach(function(row) {
|
|
var wd = asNullOrNumber(row.wd);
|
|
var wv = asNullOrNumber(row.wv);
|
|
var keys = sampleType ? sampleType === "all" ? validOverlays : [sampleType] : [];
|
|
var data = keys.map(function(key) {
|
|
return {overlay: key, value: asNullOrNumber(row[key])};
|
|
});
|
|
|
|
if (row.stationId == 208) {
|
|
wd = wv = null; // this station appears to have bogus wind readings
|
|
}
|
|
|
|
// skip rows that have no data
|
|
if (!(_.isFinite(wd) && _.isFinite(wv)) && data.filter(_.isFinite).length === 0) {
|
|
return;
|
|
}
|
|
|
|
var sample = {
|
|
stationId: row.stationId.toString(),
|
|
coordinates: [asNullOrNumber(row.longitude), asNullOrNumber(row.latitude)],
|
|
wind: [wd, wv]
|
|
};
|
|
data.forEach(function(datum) {
|
|
sample[datum.overlay] = datum.value;
|
|
});
|
|
|
|
var date = row.date + ":00";
|
|
var bucket = buckets[date];
|
|
if (!bucket) {
|
|
buckets[date] = bucket = [];
|
|
}
|
|
bucket.push(sample);
|
|
});
|
|
|
|
var result = [];
|
|
var mostRecent = new Date("1901-01-01 00:00:00Z");
|
|
Object.keys(buckets).forEach(function(date) {
|
|
result.push({date: tool.withZone(date, "+09:00"), samples: buckets[date]});
|
|
mostRecent = dateMax(mostRecent, new Date(date));
|
|
});
|
|
return {lastModified: mostRecent, jsonPayload: JSON.stringify(result), notFound: result.length === 0};
|
|
}
|
|
|
|
function doQuery(constraints) {
|
|
var columns = ["date", "stationId", "longitude", "latitude", "wv", "wd"];
|
|
var sampleType = constraints.sampleType;
|
|
if (sampleType) {
|
|
columns = columns.concat(sampleType === "all" ? validOverlays : [sampleType]);
|
|
}
|
|
var stmt = db.selectSamplesCompact(constraints, columns);
|
|
return db.execute(stmt).then(buildResponse.bind(null, sampleType));
|
|
}
|
|
|
|
function memoize(f, maxEntries) {
|
|
var memos = {};
|
|
|
|
function invoke(x) {
|
|
var key = JSON.stringify(arguments);
|
|
if (_.has(memos, key)) {
|
|
return memos[key];
|
|
}
|
|
|
|
var keys = Object.keys(memos);
|
|
if (keys.length >= maxEntries) { // If too many memos, just remove a random one--it's easy.
|
|
delete memos[keys[_.random(0, keys.length - 1)]];
|
|
}
|
|
|
|
return memos[key] = f.apply(this, arguments);
|
|
}
|
|
|
|
invoke.resetMemos = function() { memos = {}; };
|
|
return invoke;
|
|
}
|
|
|
|
var memoizedQuery = memoize(doQuery, 100); // Allow many memos, but don't grow indefinitely.
|
|
exports.resetQueryMemos = memoizedQuery.resetMemos;
|
|
|
|
function query(res, constraints) {
|
|
var queryTask = memoizedQuery(constraints);
|
|
|
|
function sendResponse(data) {
|
|
if (data.notFound) {
|
|
prepareCacheControl(res, 30);
|
|
return res.send(404);
|
|
}
|
|
prepareLastModified(res, data.lastModified);
|
|
res.set("Content-Type", "application/json");
|
|
res.send(data.jsonPayload);
|
|
}
|
|
|
|
return queryTask.then(sendResponse).then(null, handleUnexpected.bind(null, res));
|
|
}
|
|
|
|
function isValid(sampleType) {
|
|
return sampleType === "wind" ||
|
|
sampleType === "all" ||
|
|
sampleType !== "stationId" && sampleType !== "date" && _.contains(validSampleTypes, sampleType);
|
|
}
|
|
|
|
app.get("/data/:type/current", function(req, res) {
|
|
try {
|
|
var sampleType = req.params.type;
|
|
if (!isValid(sampleType)) {
|
|
return res.send(400);
|
|
}
|
|
var constraints = {date: {current: true, parts: [], zone: "+09:00"}};
|
|
if (sampleType !== "wind") {
|
|
constraints.sampleType = sampleType;
|
|
}
|
|
query(res, constraints);
|
|
}
|
|
catch (error) {
|
|
handleUnexpected(res, error);
|
|
}
|
|
});
|
|
|
|
app.get("/data/:type/:year/:month", function(req, res) {
|
|
try {
|
|
var sampleType = req.params.type;
|
|
var parts = parseDateParts(req.params.year, req.params.month).filter(_.isFinite);
|
|
if (!isValid(sampleType) || isNaN(parts[0]) || isNaN(parts[1])) {
|
|
return res.send(400);
|
|
}
|
|
var constraints = {date: {current: false, parts: parts, zone: "+09:00"}};
|
|
if (sampleType !== "wind") {
|
|
constraints.sampleType = sampleType;
|
|
}
|
|
query(res, constraints);
|
|
}
|
|
catch (error) {
|
|
handleUnexpected(res, error);
|
|
}
|
|
});
|
|
|
|
app.get("/data/:type/:year/:month/:day", function(req, res) {
|
|
try {
|
|
var sampleType = req.params.type;
|
|
var parts = parseDateParts(req.params.year, req.params.month, req.params.day).filter(_.isFinite);
|
|
if (!isValid(sampleType) || isNaN(parts[0]) || isNaN(parts[1]) || isNaN(parts[2])) {
|
|
return res.send(400);
|
|
}
|
|
var constraints = {date: {current: false, parts: parts, zone: "+09:00"}};
|
|
if (sampleType !== "wind") {
|
|
constraints.sampleType = sampleType;
|
|
}
|
|
query(res, constraints);
|
|
}
|
|
catch (error) {
|
|
handleUnexpected(res, error);
|
|
}
|
|
});
|
|
|
|
app.get("/data/:type/:year/:month/:day/:hour", function(req, res) {
|
|
try {
|
|
var sampleType = req.params.type;
|
|
var parts = parseDateParts(req.params.year, req.params.month, req.params.day, req.params.hour);
|
|
if (!isValid(sampleType) || isNaN(parts[0]) || isNaN(parts[1]) || isNaN(parts[2]) || isNaN(parts[3])) {
|
|
return res.send(400);
|
|
}
|
|
var constraints = {date: {current: false, parts: parts, zone: "+09:00"}};
|
|
if (sampleType !== "wind") {
|
|
constraints.sampleType = sampleType;
|
|
}
|
|
query(res, constraints);
|
|
}
|
|
catch (error) {
|
|
handleUnexpected(res, error);
|
|
}
|
|
});
|
|
|
|
var samplesRegex = /data-samples="\/data\/wind\/current"/; // index.html parameter for samples url
|
|
var typeRegex = /data-type="wind"/; // index.html parameter for sample type
|
|
var dateRegex = /data-date=""/; // index.html parameter for date of the samples, when specified
|
|
|
|
app.get("/map/:type/current", function(req, res) {
|
|
try {
|
|
var sampleType = req.params.type;
|
|
if (!isValid(sampleType)) {
|
|
return res.send(400);
|
|
}
|
|
|
|
// Perform replacements -- should probably use a proper templating engine...
|
|
var text = indexHTMLText;
|
|
text = text.replace(typeRegex, tool.format('data-type="{0}"', sampleType));
|
|
text = text.replace(samplesRegex, tool.format('data-samples="/data/{0}/current"', sampleType));
|
|
|
|
prepareLastModified(res, indexHTMLDate);
|
|
res.send(text);
|
|
}
|
|
catch (error) {
|
|
handleUnexpected(res, error);
|
|
}
|
|
});
|
|
|
|
app.get("/map/:type/:year/:month/:day/:hour", function(req, res) {
|
|
try {
|
|
var sampleType = req.params.type;
|
|
var parts = parseDateParts(req.params.year, req.params.month, req.params.day, req.params.hour);
|
|
if (!isValid(sampleType) || isNaN(parts[0]) || isNaN(parts[1]) || isNaN(parts[2]) || isNaN(parts[3])) {
|
|
return res.send(400);
|
|
}
|
|
var date = tool.toISOString({year: parts[0], month: parts[1], day: parts[2], hour: parts[3]});
|
|
|
|
// Perform replacements -- should probably use a proper templating engine...
|
|
var text = indexHTMLText;
|
|
text = text.replace(typeRegex, tool.format('data-type="{0}"', sampleType));
|
|
text = text.replace(samplesRegex, tool.format('data-samples="/data/{0}/{1}"', sampleType, parts.join("/")));
|
|
text = text.replace(dateRegex, tool.format('data-date="{0}"', date.substr(0, date.length - 1))); // remove 'Z'
|
|
|
|
prepareLastModified(res, indexHTMLDate);
|
|
res.send(text);
|
|
}
|
|
catch (error) {
|
|
handleUnexpected(res, error);
|
|
}
|
|
});
|
|
|
|
// CF won't compress MIME type "application/x-font-ttf" (the express.js default) but will compress "font/ttf".
|
|
// https://support.cloudflare.com/hc/en-us/articles/200168396-What-will-CloudFlare-gzip-
|
|
express.static.mime.define({"font/ttf": ["ttf"]});
|
|
|
|
app.use(express.static(__dirname + "/public"));
|
|
|
|
/**
|
|
* Adds headers to a response to specify the last modified date.
|
|
*/
|
|
function prepareLastModified(res, lastModified) {
|
|
res.set("Last-Modified", lastModified.toUTCString());
|
|
}
|
|
|
|
/**
|
|
* Adds headers to a response to enable caching. maxAge is number of seconds to cache the response.
|
|
*/
|
|
function prepareCacheControl(res, maxAge) {
|
|
res.setHeader("Cache-Control", "public, max-age=" + maxAge);
|
|
if (maxAge) {
|
|
var now = (Math.ceil(Date.now() / 1000) + 1) * 1000;
|
|
res.setHeader("Expires", new Date(now + maxAge * 1000).toUTCString());
|
|
}
|
|
}
|
|
|
|
function cacheControl() {
|
|
var SECOND = 1;
|
|
var MINUTE = 60 * SECOND;
|
|
var HOUR = 60 * MINUTE;
|
|
var DAY = 24 * HOUR;
|
|
var DEFAULT = 30 * MINUTE;
|
|
|
|
var rules = [
|
|
// very-short-lived
|
|
[/data\/.*\/current/, 1 * MINUTE],
|
|
|
|
// short-lived (default behavior for all other resources)
|
|
[/js\/air\.js/, DEFAULT], // override medium-lived .js rule below
|
|
[/js\/mvi\.js/, DEFAULT], // override medium-lived .js rule below
|
|
|
|
// medium-lived
|
|
[/js\/.*\.js/, 5 * DAY],
|
|
[/tokyo-topo\.json/, 5 * DAY],
|
|
|
|
// long-lived
|
|
[/mplus-.*\.ttf/, 30 * DAY],
|
|
[/\.png|\.ico/, 30 * DAY]
|
|
];
|
|
|
|
return function(req, res, next) {
|
|
var maxAge = DEFAULT;
|
|
for (var i = 0; i < rules.length; i++) {
|
|
var rule = rules[i];
|
|
if (rule[0].test(req.url)) {
|
|
maxAge = rule[1];
|
|
break;
|
|
}
|
|
}
|
|
prepareCacheControl(res, maxAge);
|
|
return next();
|
|
};
|
|
}
|
|
|
|
function readJsonSync(path) {
|
|
return JSON.parse(fs.readFileSync(path, {encoding: "utf8"}));
|
|
}
|
|
|
|
function cors() {
|
|
|
|
var rules = [];
|
|
if (fs.existsSync(ruleFile)) {
|
|
rules = readJsonSync(ruleFile).map(function(entry) {
|
|
return new RegExp(entry);
|
|
});
|
|
}
|
|
|
|
return function(req, res, next) {
|
|
for (var i = 0; i < rules.length; i++) {
|
|
if (rules[i].test(req.headers.origin)) {
|
|
res.header("Access-Control-Allow-Origin", req.headers.origin);
|
|
break;
|
|
}
|
|
}
|
|
return next();
|
|
}
|
|
}
|
|
|
|
app.listen(port);
|
|
log.info(tool.format("Listening on port {0}...", port));
|