feature: add exponential backoff restart delay via --exp-backoff-restart-delay

bin/pm2

@@ -58,6 +58,7 @@ commander.version(pkg.version)
   .option('--listen-timeout <delay>', 'listen timeout on application reload')
   .option('--max-memory-restart <memory>', 'Restart the app if an amount of memory is exceeded (in bytes)')
   .option('--restart-delay <delay>', 'specify a delay between restarts (in milliseconds)')
+  .option('--exp-backoff-restart-delay <delay>', 'specify a delay between restarts (in milliseconds)')
   .option('-x --execute-command', 'execute a program using fork system')
   .option('--max-restarts [count]', 'only restart the script COUNT times')
   .option('-u --user <username>', 'define user when generating startup script')

constants.js

@@ -48,6 +48,7 @@ var csts = {
   ONLINE_STATUS           : 'online',
   STOPPED_STATUS          : 'stopped',
   STOPPING_STATUS         : 'stopping',
+  WAITING_RESTART         : 'waiting restart',
   LAUNCHING_STATUS        : 'launching',
   ERRORED_STATUS          : 'errored',
   ONE_LAUNCH_STATUS       : 'one-launch-status',

lib/API/Startup.js

@@ -448,6 +448,7 @@ module.exports = function(CLI) {
         if (!apps[0]) return fin(null);
         delete apps[0].pm2_env.instances;
         delete apps[0].pm2_env.pm_id;
+        delete apps[0].pm2_env.prev_restart_delay;
         if (!apps[0].pm2_env.pmx_module)
           env_arr.push(apps[0].pm2_env);
         apps.shift();

lib/API/schema.json

@@ -111,6 +111,11 @@
     "docDefault": 0,
     "docDescription": "Time in ms to wait before restarting a crashing app"
   },
+  "exp_backoff_restart_delay": {
+    "type": "number",
+    "docDefault": 0,
+    "docDescription": "Restart Time in ms to wait before restarting a crashing app"
+  },
   "source_map_support": {
     "type": "boolean",
     "docDefault": true,

lib/God.js

@@ -147,7 +147,7 @@ God.executeApp = function executeApp(env, cb) {
         God.notify('online', proc);
 
       proc.pm2_env.status = cst.ONLINE_STATUS;
-      console.log('App name:%s id:%s online', proc.pm2_env.name, proc.pm2_env.pm_id);
+      console.log(`App [${proc.pm2_env.name}:${proc.pm2_env.pm_id}] online`);
       if (cb) cb(null, proc);
   }
 
@@ -281,7 +281,7 @@ God.executeApp = function executeApp(env, cb) {
  * @return
  */
 God.handleExit = function handleExit(clu, exit_code, kill_signal) {
-  console.log('App [%s] with id [%s] and pid [%s], exited with code [%s] via signal [%s]', clu.pm2_env.name, clu.pm2_env.pm_id, clu.process.pid, exit_code, kill_signal || 'SIGINT');
+  console.log(`App [${clu.pm2_env.name}:${clu.pm2_env.pm_id}] exited with code [${exit_code}] via signal [${kill_signal || 'SIGINT'}]`)
 
   var proc = this.clusters_db[clu.pm2_env.pm_id];
 
@@ -361,10 +361,27 @@ God.handleExit = function handleExit(clu, exit_code, kill_signal) {
   }
 
   var restart_delay = 0;
-  if (proc.pm2_env.restart_delay !== undefined && !isNaN(parseInt(proc.pm2_env.restart_delay))) {
+
+  if (proc.pm2_env.restart_delay !== undefined &&
+      !isNaN(parseInt(proc.pm2_env.restart_delay))) {
+    proc.pm2_env.status = cst.WAITING_RESTART;
     restart_delay = parseInt(proc.pm2_env.restart_delay);
   }
 
+  if (proc.pm2_env.exp_backoff_restart_delay !== undefined &&
+      !isNaN(parseInt(proc.pm2_env.exp_backoff_restart_delay))) {
+    proc.pm2_env.status = cst.WAITING_RESTART;
+    if (!proc.pm2_env.prev_restart_delay) {
+      proc.pm2_env.prev_restart_delay = proc.pm2_env.exp_backoff_restart_delay
+      restart_delay = proc.pm2_env.exp_backoff_restart_delay
+    }
+    else {
+      proc.pm2_env.prev_restart_delay = Math.floor(Math.min(15000, proc.pm2_env.prev_restart_delay * 1.5))
+      restart_delay = proc.pm2_env.prev_restart_delay
+    }
+    console.log(`App [${clu.pm2_env.name}:${clu.pm2_env.pm_id}] will restart in ${restart_delay}ms`)
+  }
+
   if (!stopping && !overlimit) {
     //make this property unenumerable
     Object.defineProperty(proc.pm2_env, 'restart_task', {configurable: true, writable: true});

lib/God/ClusterMode.js

@@ -33,10 +33,7 @@ module.exports = function ClusterMode(God) {
   God.nodeApp = function nodeApp(env_copy, cb){
     var clu = null;
 
-    console.log('Starting execution sequence in -cluster mode- for app name:%s id:%s',
-                env_copy.name,
-                env_copy.pm_id);
-
+    console.log(`App [${env_copy.name}:${env_copy.pm_id}] starting in -cluster mode-`)
     if (env_copy.node_args && Array.isArray(env_copy.node_args)) {
       cluster.settings.execArgv = env_copy.node_args;
     }

lib/God/ForkMode.js

@@ -35,9 +35,7 @@ module.exports = function ForkMode(God) {
     var command = '';
     var args    = [];
 
-    console.log('Starting execution sequence in -fork mode- for app name:%s id:%s',
-                pm2_env.name,
-                pm2_env.pm_id);
+    console.log(`App [${pm2_env.name}:${pm2_env.pm_id}] starting in -fork mode-`)
     var spawn = require('child_process').spawn;
 
     var interpreter = pm2_env.exec_interpreter || 'node';

lib/God/Methods.js

@@ -243,6 +243,7 @@ module.exports = function(God) {
   God.resetState = function(pm2_env) {
     pm2_env.created_at = Date.now();
     pm2_env.unstable_restarts = 0;
+    pm2_env.prev_restart_delay = 0;
   };
 
 };

test/programmatic/exp_backoff_restart_delay.mocha.js

@@ -0,0 +1,53 @@
+
+const PM2 = require('../..');
+const should = require('should');
+const exec = require('child_process').exec
+const path = require('path')
+
+describe('Exponential backoff feature', function() {
+  var pm2
+  var test_path = path.join(__dirname, 'fixtures', 'exp-backoff')
+
+  after(function(done) {
+    pm2.delete('all', function() {
+      pm2.kill(done);
+    })
+  });
+
+  before(function(done) {
+    pm2 = new PM2.custom({
+      cwd : test_path
+    });
+
+    pm2.delete('all', () => done())
+  })
+
+  it('should set exponential backoff restart', (done) => {
+    pm2.start({
+      script: path.join(test_path, 'throw.js'),
+      exp_backoff_restart_delay: 100
+    }, (err, apps) => {
+      should(err).be.null()
+      should(apps[0].pm2_env.exp_backoff_restart_delay).eql(100)
+      done()
+    })
+  })
+
+  it('should have set the prev_restart delay', (done) => {
+    setTimeout(() => {
+      pm2.list((err, procs) => {
+        should(procs[0].pm2_env.prev_restart_delay).eql(100)
+        done()
+      })
+    }, 100)
+  })
+
+  it('should have incremented the prev_restart delay', (done) => {
+    setTimeout(() => {
+      pm2.list((err, procs) => {
+        should(procs[0].pm2_env.prev_restart_delay).be.above(100)
+        done()
+      })
+    }, 300)
+  })
+})

test/programmatic/fixtures/exp-backoff/throw.js

@@ -0,0 +1 @@
+throw new Error('Ugly error')

test/unit.sh

@@ -50,6 +50,8 @@ spec "Reload locker tests"
 
 mocha --exit --opts ./mocha.opts  ./version.mocha.js
 spec "Package json version retriever"
+mocha --exit --opts ./mocha.opts  ./exp_backoff_restart_delay.mocha.js
+spec "Exponential backoff restart delay tests"
 
 mocha --exit --opts ./mocha.opts  ./api.backward.compatibility.mocha.js
 spec "API Backward compatibility tests"