diff --git a/bin/pm2 b/bin/pm2 index fc1c268b..bc0f0857 100755 --- a/bin/pm2 +++ b/bin/pm2 @@ -58,6 +58,7 @@ commander.version(pkg.version) .option('--listen-timeout ', 'listen timeout on application reload') .option('--max-memory-restart ', 'Restart the app if an amount of memory is exceeded (in bytes)') .option('--restart-delay ', 'specify a delay between restarts (in milliseconds)') + .option('--exp-backoff-restart-delay ', 'specify a delay between restarts (in milliseconds)') .option('-x --execute-command', 'execute a program using fork system') .option('--max-restarts [count]', 'only restart the script COUNT times') .option('-u --user ', 'define user when generating startup script') diff --git a/constants.js b/constants.js index 277a3094..718cd161 100644 --- a/constants.js +++ b/constants.js @@ -48,6 +48,7 @@ var csts = { ONLINE_STATUS : 'online', STOPPED_STATUS : 'stopped', STOPPING_STATUS : 'stopping', + WAITING_RESTART : 'waiting restart', LAUNCHING_STATUS : 'launching', ERRORED_STATUS : 'errored', ONE_LAUNCH_STATUS : 'one-launch-status', diff --git a/lib/API/Startup.js b/lib/API/Startup.js index a22c934a..23103753 100644 --- a/lib/API/Startup.js +++ b/lib/API/Startup.js @@ -448,6 +448,7 @@ module.exports = function(CLI) { if (!apps[0]) return fin(null); delete apps[0].pm2_env.instances; delete apps[0].pm2_env.pm_id; + delete apps[0].pm2_env.prev_restart_delay; if (!apps[0].pm2_env.pmx_module) env_arr.push(apps[0].pm2_env); apps.shift(); diff --git a/lib/API/schema.json b/lib/API/schema.json index 8a5a664f..ab101f64 100644 --- a/lib/API/schema.json +++ b/lib/API/schema.json @@ -111,6 +111,11 @@ "docDefault": 0, "docDescription": "Time in ms to wait before restarting a crashing app" }, + "exp_backoff_restart_delay": { + "type": "number", + "docDefault": 0, + "docDescription": "Restart Time in ms to wait before restarting a crashing app" + }, "source_map_support": { "type": "boolean", "docDefault": true, diff --git a/lib/God.js b/lib/God.js index 16792786..4d1633e2 100644 --- a/lib/God.js +++ b/lib/God.js @@ -147,7 +147,7 @@ God.executeApp = function executeApp(env, cb) { God.notify('online', proc); proc.pm2_env.status = cst.ONLINE_STATUS; - console.log('App name:%s id:%s online', proc.pm2_env.name, proc.pm2_env.pm_id); + console.log(`App [${proc.pm2_env.name}:${proc.pm2_env.pm_id}] online`); if (cb) cb(null, proc); } @@ -281,7 +281,7 @@ God.executeApp = function executeApp(env, cb) { * @return */ God.handleExit = function handleExit(clu, exit_code, kill_signal) { - console.log('App [%s] with id [%s] and pid [%s], exited with code [%s] via signal [%s]', clu.pm2_env.name, clu.pm2_env.pm_id, clu.process.pid, exit_code, kill_signal || 'SIGINT'); + console.log(`App [${clu.pm2_env.name}:${clu.pm2_env.pm_id}] exited with code [${exit_code}] via signal [${kill_signal || 'SIGINT'}]`) var proc = this.clusters_db[clu.pm2_env.pm_id]; @@ -361,10 +361,27 @@ God.handleExit = function handleExit(clu, exit_code, kill_signal) { } var restart_delay = 0; - if (proc.pm2_env.restart_delay !== undefined && !isNaN(parseInt(proc.pm2_env.restart_delay))) { + + if (proc.pm2_env.restart_delay !== undefined && + !isNaN(parseInt(proc.pm2_env.restart_delay))) { + proc.pm2_env.status = cst.WAITING_RESTART; restart_delay = parseInt(proc.pm2_env.restart_delay); } + if (proc.pm2_env.exp_backoff_restart_delay !== undefined && + !isNaN(parseInt(proc.pm2_env.exp_backoff_restart_delay))) { + proc.pm2_env.status = cst.WAITING_RESTART; + if (!proc.pm2_env.prev_restart_delay) { + proc.pm2_env.prev_restart_delay = proc.pm2_env.exp_backoff_restart_delay + restart_delay = proc.pm2_env.exp_backoff_restart_delay + } + else { + proc.pm2_env.prev_restart_delay = Math.floor(Math.min(15000, proc.pm2_env.prev_restart_delay * 1.5)) + restart_delay = proc.pm2_env.prev_restart_delay + } + console.log(`App [${clu.pm2_env.name}:${clu.pm2_env.pm_id}] will restart in ${restart_delay}ms`) + } + if (!stopping && !overlimit) { //make this property unenumerable Object.defineProperty(proc.pm2_env, 'restart_task', {configurable: true, writable: true}); diff --git a/lib/God/ClusterMode.js b/lib/God/ClusterMode.js index 677f0870..e301daf2 100644 --- a/lib/God/ClusterMode.js +++ b/lib/God/ClusterMode.js @@ -33,10 +33,7 @@ module.exports = function ClusterMode(God) { God.nodeApp = function nodeApp(env_copy, cb){ var clu = null; - console.log('Starting execution sequence in -cluster mode- for app name:%s id:%s', - env_copy.name, - env_copy.pm_id); - + console.log(`App [${env_copy.name}:${env_copy.pm_id}] starting in -cluster mode-`) if (env_copy.node_args && Array.isArray(env_copy.node_args)) { cluster.settings.execArgv = env_copy.node_args; } diff --git a/lib/God/ForkMode.js b/lib/God/ForkMode.js index c511e437..fb5a1f17 100644 --- a/lib/God/ForkMode.js +++ b/lib/God/ForkMode.js @@ -35,9 +35,7 @@ module.exports = function ForkMode(God) { var command = ''; var args = []; - console.log('Starting execution sequence in -fork mode- for app name:%s id:%s', - pm2_env.name, - pm2_env.pm_id); + console.log(`App [${pm2_env.name}:${pm2_env.pm_id}] starting in -fork mode-`) var spawn = require('child_process').spawn; var interpreter = pm2_env.exec_interpreter || 'node'; diff --git a/lib/God/Methods.js b/lib/God/Methods.js index 73e0c589..04534ba2 100644 --- a/lib/God/Methods.js +++ b/lib/God/Methods.js @@ -243,6 +243,7 @@ module.exports = function(God) { God.resetState = function(pm2_env) { pm2_env.created_at = Date.now(); pm2_env.unstable_restarts = 0; + pm2_env.prev_restart_delay = 0; }; }; diff --git a/test/programmatic/exp_backoff_restart_delay.mocha.js b/test/programmatic/exp_backoff_restart_delay.mocha.js new file mode 100644 index 00000000..e6afb2b2 --- /dev/null +++ b/test/programmatic/exp_backoff_restart_delay.mocha.js @@ -0,0 +1,53 @@ + +const PM2 = require('../..'); +const should = require('should'); +const exec = require('child_process').exec +const path = require('path') + +describe('Exponential backoff feature', function() { + var pm2 + var test_path = path.join(__dirname, 'fixtures', 'exp-backoff') + + after(function(done) { + pm2.delete('all', function() { + pm2.kill(done); + }) + }); + + before(function(done) { + pm2 = new PM2.custom({ + cwd : test_path + }); + + pm2.delete('all', () => done()) + }) + + it('should set exponential backoff restart', (done) => { + pm2.start({ + script: path.join(test_path, 'throw.js'), + exp_backoff_restart_delay: 100 + }, (err, apps) => { + should(err).be.null() + should(apps[0].pm2_env.exp_backoff_restart_delay).eql(100) + done() + }) + }) + + it('should have set the prev_restart delay', (done) => { + setTimeout(() => { + pm2.list((err, procs) => { + should(procs[0].pm2_env.prev_restart_delay).eql(100) + done() + }) + }, 100) + }) + + it('should have incremented the prev_restart delay', (done) => { + setTimeout(() => { + pm2.list((err, procs) => { + should(procs[0].pm2_env.prev_restart_delay).be.above(100) + done() + }) + }, 300) + }) +}) diff --git a/test/programmatic/fixtures/exp-backoff/throw.js b/test/programmatic/fixtures/exp-backoff/throw.js new file mode 100644 index 00000000..89c51b6e --- /dev/null +++ b/test/programmatic/fixtures/exp-backoff/throw.js @@ -0,0 +1 @@ +throw new Error('Ugly error') diff --git a/test/unit.sh b/test/unit.sh index 078dce40..ea462e26 100644 --- a/test/unit.sh +++ b/test/unit.sh @@ -50,6 +50,8 @@ spec "Reload locker tests" mocha --exit --opts ./mocha.opts ./version.mocha.js spec "Package json version retriever" +mocha --exit --opts ./mocha.opts ./exp_backoff_restart_delay.mocha.js +spec "Exponential backoff restart delay tests" mocha --exit --opts ./mocha.opts ./api.backward.compatibility.mocha.js spec "API Backward compatibility tests"