From cd2dff720260ff41e71f43bd11517be06602b48a Mon Sep 17 00:00:00 2001
From: Tom Ashworth <tom@phuu.net>
Date: Wed, 9 Jan 2013 10:23:37 +0000
Subject: [PATCH] Chunk the owners table to populate it slowly.

---
 build/populate.js         | 29 ++++++++++++++++++++++++-----
 lib/db/mysql.js           |  4 ++++
 lib/db/sql_templates.json |  1 +
 lib/store.js              |  3 ++-
 4 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/build/populate.js b/build/populate.js
index 50b738fd..aee6835c 100644
--- a/build/populate.js
+++ b/build/populate.js
@@ -24,6 +24,7 @@ var populateBin = function (ownerBin, done) {
   
   getBin(ownerBin, function (err, sandboxBin) {
     if (err) return done(err);
+    if (!sandboxBin) return done();
 
     ownerBin.summary = utils.titleForBin(sandboxBin);
 
@@ -35,10 +36,28 @@ var populateBin = function (ownerBin, done) {
 
 // Start
 
-store.getAllOwners(function (err, results) {
-  if (err) return console.error('getAllOwners:', err);
+var start = 0,
+    completed = 0,
+    blocksize = 150;
 
-  async.forEach(results, populateBin, function (err) {
-    if (err) return console.error('async done:', err);
+var populate = function () {
+  store.getOwnersBlock(start, blocksize, function (err, owners) {
+    if (err) return console.error('getAllOwners:', err);
+
+    async.forEachSeries(owners, populateBin, function (err) {
+      if (err) return console.error('async done:', err);
+
+      completed += owners.length;
+
+      if (owners.length < blocksize) {
+        console.log('===== done %d', completed);
+      } else {
+        console.log('===== block %d', completed);
+        start += blocksize;
+        setTimeout(populate, 1000 * 0.5);
+      }
+    });
   });
-});
\ No newline at end of file
+};
+
+populate();
\ No newline at end of file
diff --git a/lib/db/mysql.js b/lib/db/mysql.js
index 0fc2fb60..ba1c1f34 100644
--- a/lib/db/mysql.js
+++ b/lib/db/mysql.js
@@ -175,6 +175,10 @@ module.exports = utils.inherit(Object, {
     // Get all the 'owned' bins
     this.connection.query(templates.getAllOwners, [], fn);
   },
+  getOwnersBlock: function (start, size, fn) {
+    // Get all the 'owned' bins
+    this.connection.query(templates.getOwnersBlock, [start, size], fn);
+  },
   generateBinId: function (fn, attempts) {
     var id = utils.shortcode(), mysql = this;
 
diff --git a/lib/db/sql_templates.json b/lib/db/sql_templates.json
index e22995c6..df35a18b 100644
--- a/lib/db/sql_templates.json
+++ b/lib/db/sql_templates.json
@@ -14,6 +14,7 @@
   "DISABLEDgetBinsByUser": "SELECT SUBSTR(s.html, 1, 200) as html, SUBSTR(s.javascript, 1, 100) as javascript, SUBSTR(s.css, 1, 100) as css, o.last_updated as created, o.url, o.revision FROM `owners` as o, `sandbox` s WHERE o.url=s.url and o.revision=s.revision and o.name=?",
   "getBinsByUser": "SELECT *, `last_updated` as `created` FROM `owners` WHERE `name`=?",
   "getAllOwners": "SELECT * FROM `owners`",
+  "getOwnersBlock": "SELECT * FROM `owners` LIMIT ?, ?",
   "getBinByUrlAndRevision": "SELECT * FROM `sandbox` WHERE `url`=? AND `revision`=? LIMIT 1",
   "getLatestBinForUser": "SELECT `url`, `revision` FROM `owners` WHERE `name`=? AND `last_updated` != 0 ORDER BY `last_updated` DESC LIMIT 1",
   "touchOwnership": "UPDATE `owners` SET `last_updated`=?, `summary`=? WHERE `name`=? AND `url`=? AND `revision` = ?",
diff --git a/lib/store.js b/lib/store.js
index a6859500..88f49f64 100644
--- a/lib/store.js
+++ b/lib/store.js
@@ -28,7 +28,8 @@ var methods = [
   'expireForgotToken',
   'expireForgotTokenByUser',
   'reportBin',
-  'getAllOwners'
+  'getAllOwners',
+  'getOwnersBlock'
 ];
 
 // Proxy the methods through the store.