archive-gh-me/jsbin
1
0
Fork 0
mirror of https://github.com/jsbin/jsbin.git synced 2026-01-25 15:38:56 +00:00
Code Issues Projects Releases Wiki Activity

Chunk the owners table to populate it slowly.

Browse Source
This commit is contained in:
Tom Ashworth 2013-01-09 10:23:37 +00:00
parent 806e1aaea9
commit cd2dff7202
4 changed files with 31 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
build/populate.js
View File

@ -24,6 +24,7 @@ var populateBin = function (ownerBin, done) {
getBin(ownerBin, function (err, sandboxBin) {
if (err) return done(err);
if (!sandboxBin) return done();
ownerBin.summary = utils.titleForBin(sandboxBin);
@ -35,10 +36,28 @@ var populateBin = function (ownerBin, done) {
// Start
store.getAllOwners(function (err, results) {
if (err) return console.error('getAllOwners:', err);
var start = 0,
completed = 0,
blocksize = 150;
async.forEach(results, populateBin, function (err) {
if (err) return console.error('async done:', err);
var populate = function () {
store.getOwnersBlock(start, blocksize, function (err, owners) {
if (err) return console.error('getAllOwners:', err);
async.forEachSeries(owners, populateBin, function (err) {
if (err) return console.error('async done:', err);
completed += owners.length;
if (owners.length < blocksize) {
console.log('===== done %d', completed);
} else {
console.log('===== block %d', completed);
start += blocksize;
setTimeout(populate, 1000 * 0.5);
}
});
});
});
};
populate();

4
lib/db/mysql.js
View File

@ -175,6 +175,10 @@ module.exports = utils.inherit(Object, {
// Get all the 'owned' bins
this.connection.query(templates.getAllOwners, [], fn);
},
getOwnersBlock: function (start, size, fn) {
// Get all the 'owned' bins
this.connection.query(templates.getOwnersBlock, [start, size], fn);
},
generateBinId: function (fn, attempts) {
var id = utils.shortcode(), mysql = this;

1
lib/db/sql_templates.json
View File

@ -14,6 +14,7 @@
"DISABLEDgetBinsByUser": "SELECT SUBSTR(s.html, 1, 200) as html, SUBSTR(s.javascript, 1, 100) as javascript, SUBSTR(s.css, 1, 100) as css, o.last_updated as created, o.url, o.revision FROM `owners` as o, `sandbox` s WHERE o.url=s.url and o.revision=s.revision and o.name=?",
"getBinsByUser": "SELECT *, `last_updated` as `created` FROM `owners` WHERE `name`=?",
"getAllOwners": "SELECT * FROM `owners`",
"getOwnersBlock": "SELECT * FROM `owners` LIMIT ?, ?",
"getBinByUrlAndRevision": "SELECT * FROM `sandbox` WHERE `url`=? AND `revision`=? LIMIT 1",
"getLatestBinForUser": "SELECT `url`, `revision` FROM `owners` WHERE `name`=? AND `last_updated` != 0 ORDER BY `last_updated` DESC LIMIT 1",
"touchOwnership": "UPDATE `owners` SET `last_updated`=?, `summary`=? WHERE `name`=? AND `url`=? AND `revision` = ?",

3
lib/store.js
View File

@ -28,7 +28,8 @@ var methods = [
'expireForgotToken',
'expireForgotTokenByUser',
'reportBin',
'getAllOwners'
'getAllOwners',
'getOwnersBlock'
];
// Proxy the methods through the store.