diff --git a/lib/application.js b/lib/application.js
index 25857288f..eac1e07ad 100644
--- a/lib/application.js
+++ b/lib/application.js
@@ -173,12 +173,11 @@ class Application extends EggApplication {
     if (!this[KEYS]) {
       if (!this.config.keys) {
         if (this.config.env === 'local' || this.config.env === 'unittest') {
-          console.warn('Please set config.keys first, now using mock keys for dev env (%s)',
-            this.config.baseDir);
-          this.config.keys = 'foo, keys, you need to set your app keys';
-        } else {
-          throw new Error('Please set config.keys first');
+          const configPath = path.join(this.config.baseDir, 'config/config.default.js');
+          console.error('Cookie need secret key to sign and encrypt.');
+          console.error('Please add `config.keys` in %s', configPath);
         }
+        throw new Error('Please set config.keys first');
       }
 
       this[KEYS] = this.config.keys.split(',').map(s => s.trim());
diff --git a/test/fixtures/apps/agent-app-sync/config/config.default.js b/test/fixtures/apps/agent-app-sync/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/agent-app-sync/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/agent-app/config/config.default.js b/test/fixtures/apps/agent-app/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/agent-app/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/agent-client-app/config/config.default.js b/test/fixtures/apps/agent-client-app/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/agent-client-app/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/agent-die/config/config.default.js b/test/fixtures/apps/agent-die/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/agent-die/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/agent-restart/config/config.default.js b/test/fixtures/apps/agent-restart/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/agent-restart/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/aliyun-egg/config/config.default.js b/test/fixtures/apps/aliyun-egg/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/aliyun-egg/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/app-router/config/config.default.js b/test/fixtures/apps/app-router/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/app-router/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/app-throw/config/config.default.js b/test/fixtures/apps/app-throw/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/app-throw/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/cluster-client-error/config/config.default.js b/test/fixtures/apps/cluster-client-error/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/cluster-client-error/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/context_httpclient/config/config.default.js b/test/fixtures/apps/context_httpclient/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/context_httpclient/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/custom-logger/config/config.default.js b/test/fixtures/apps/custom-logger/config/config.default.js
index 77c1855b0..722df076f 100644
--- a/test/fixtures/apps/custom-logger/config/config.default.js
+++ b/test/fixtures/apps/custom-logger/config/config.default.js
@@ -10,5 +10,6 @@ module.exports = info => {
         formatter: meta => meta.message,
       },
     },
+    keys: 'test key',
   };
 };
diff --git a/test/fixtures/apps/dnscache_httpclient/config/config.default.js b/test/fixtures/apps/dnscache_httpclient/config/config.default.js
index ce37b4030..18a819f3d 100644
--- a/test/fixtures/apps/dnscache_httpclient/config/config.default.js
+++ b/test/fixtures/apps/dnscache_httpclient/config/config.default.js
@@ -3,3 +3,5 @@
 exports.httpclient = {
   enableDNSCache: true,
 };
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/dumpconfig/config/config.default.js b/test/fixtures/apps/dumpconfig/config/config.default.js
index 70c69df07..958de9019 100644
--- a/test/fixtures/apps/dumpconfig/config/config.default.js
+++ b/test/fixtures/apps/dumpconfig/config/config.default.js
@@ -1 +1,3 @@
 exports.dynamic = 0;
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/empty/config/config.default.js b/test/fixtures/apps/empty/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/empty/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/get-logger/config/config.js b/test/fixtures/apps/get-logger/config/config.js
index 53fba40f4..edc57b712 100644
--- a/test/fixtures/apps/get-logger/config/config.js
+++ b/test/fixtures/apps/get-logger/config/config.js
@@ -9,5 +9,7 @@ module.exports = appInfo => {
         file: path.join(appInfo.baseDir, 'logs', appInfo.name, 'a.log'),
       },
     },
+
+    keys: 'secret key',
   };
 };
diff --git a/test/fixtures/apps/helper/config/config.default.js b/test/fixtures/apps/helper/config/config.default.js
index 3d49e31cb..32d25893a 100644
--- a/test/fixtures/apps/helper/config/config.default.js
+++ b/test/fixtures/apps/helper/config/config.default.js
@@ -9,3 +9,5 @@ exports.helpers = {
     }
   },
 };
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/i18n/config/config.default.js b/test/fixtures/apps/i18n/config/config.default.js
index 691fd2aa4..843dca0f5 100644
--- a/test/fixtures/apps/i18n/config/config.default.js
+++ b/test/fixtures/apps/i18n/config/config.default.js
@@ -3,3 +3,5 @@
 exports.view = {
   defaultViewEngine: 'nunjucks',
 };
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/loader-plugin-dep-missing/config/config.default.js b/test/fixtures/apps/loader-plugin-dep-missing/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/loader-plugin-dep-missing/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/loader-plugin-dep-recursive/config/config.default.js b/test/fixtures/apps/loader-plugin-dep-recursive/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/loader-plugin-dep-recursive/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/loader-plugin-dep/config/config.default.js b/test/fixtures/apps/loader-plugin-dep/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/loader-plugin-dep/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/loader-plugin-noexist/config/config.default.js b/test/fixtures/apps/loader-plugin-noexist/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/loader-plugin-noexist/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/loader-plugin/config/config.default.js b/test/fixtures/apps/loader-plugin/config/config.default.js
index cc35c7079..a464a1646 100644
--- a/test/fixtures/apps/loader-plugin/config/config.default.js
+++ b/test/fixtures/apps/loader-plugin/config/config.default.js
@@ -1,3 +1,5 @@
 exports.plugin = 'override plugin';
 
 exports.middleware = [];
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/logger-level-debug/config/config.default.js b/test/fixtures/apps/logger-level-debug/config/config.default.js
index 41a5695db..cfc99327d 100644
--- a/test/fixtures/apps/logger-level-debug/config/config.default.js
+++ b/test/fixtures/apps/logger-level-debug/config/config.default.js
@@ -3,3 +3,5 @@
 exports.logger = {
   level: 'DEBUG',
 };
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/logger-reload/config/config.default.js b/test/fixtures/apps/logger-reload/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/logger-reload/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/logger/config/config.default.js b/test/fixtures/apps/logger/config/config.default.js
index 9d132710e..d729767d1 100644
--- a/test/fixtures/apps/logger/config/config.default.js
+++ b/test/fixtures/apps/logger/config/config.default.js
@@ -12,5 +12,6 @@ module.exports = info => {
         file: path.join(info.baseDir, 'logs/custom.log'),
       },
     },
+    keys: 'test key',
   };
 };
diff --git a/test/fixtures/apps/logrotator-app/config/config.default.js b/test/fixtures/apps/logrotator-app/config/config.default.js
index 68a4709cb..34f82a9fa 100755
--- a/test/fixtures/apps/logrotator-app/config/config.default.js
+++ b/test/fixtures/apps/logrotator-app/config/config.default.js
@@ -5,3 +5,5 @@ exports.logrotator = {
   maxFiles: 2,
   rotateDuration: 30000
 };
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/master-worker-started/config/config.default.js b/test/fixtures/apps/master-worker-started/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/master-worker-started/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/messenger-app-agent/config/config.default.js b/test/fixtures/apps/messenger-app-agent/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/messenger-app-agent/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/messenger-broadcast/config/config.default.js b/test/fixtures/apps/messenger-broadcast/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/messenger-broadcast/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/messenger-random/config/config.default.js b/test/fixtures/apps/messenger-random/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/messenger-random/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/messenger/config/config.default.js b/test/fixtures/apps/messenger/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/messenger/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/multiple-view-engine/config/config.default.js b/test/fixtures/apps/multiple-view-engine/config/config.default.js
index c126f5bb8..96e99d62f 100644
--- a/test/fixtures/apps/multiple-view-engine/config/config.default.js
+++ b/test/fixtures/apps/multiple-view-engine/config/config.default.js
@@ -17,5 +17,7 @@ module.exports = appInfo => {
         '.html': 'html',
       },
     },
+
+    keys: 'test key',
   }
 };
diff --git a/test/fixtures/apps/nobuffer-logger/config/config.default.js b/test/fixtures/apps/nobuffer-logger/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/nobuffer-logger/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/notready/config/config.default.js b/test/fixtures/apps/notready/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/notready/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/reload-worker/config/config.default.js b/test/fixtures/apps/reload-worker/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/reload-worker/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/schedule/config/config.default.js b/test/fixtures/apps/schedule/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/schedule/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/service-app/config/config.default.js b/test/fixtures/apps/service-app/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/service-app/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/services_loader_verify/config/config.default.js b/test/fixtures/apps/services_loader_verify/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/services_loader_verify/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/singleton-demo/config/config.default.js b/test/fixtures/apps/singleton-demo/config/config.default.js
index cd68e0d40..0a58b73c1 100644
--- a/test/fixtures/apps/singleton-demo/config/config.default.js
+++ b/test/fixtures/apps/singleton-demo/config/config.default.js
@@ -10,3 +10,5 @@ exports.dataService = {
     foo: 'bar',
   }
 };
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/subdir-services/config/config.default.js b/test/fixtures/apps/subdir-services/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/subdir-services/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/userservice/config/config.default.js b/test/fixtures/apps/userservice/config/config.default.js
index ad74d08ac..e94f56977 100644
--- a/test/fixtures/apps/userservice/config/config.default.js
+++ b/test/fixtures/apps/userservice/config/config.default.js
@@ -16,3 +16,5 @@ exports.userservice = {
     },
   },
 };
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/view-render/config/config.default.js b/test/fixtures/apps/view-render/config/config.default.js
index aa27c0327..8fbbea53f 100644
--- a/test/fixtures/apps/view-render/config/config.default.js
+++ b/test/fixtures/apps/view-render/config/config.default.js
@@ -7,3 +7,5 @@ exports.security = {
 exports.view = {
   defaultViewEngine: 'nunjucks',
 };
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/watcher-development-app/config/config.unittest.js b/test/fixtures/apps/watcher-development-app/config/config.unittest.js
index ed9616cfb..a6fea8870 100644
--- a/test/fixtures/apps/watcher-development-app/config/config.unittest.js
+++ b/test/fixtures/apps/watcher-development-app/config/config.unittest.js
@@ -5,3 +5,5 @@ exports.env = 'local';
 exports.watcher = {
   type: 'development',
 };
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/watcher-type-default/config/config.unittest.js b/test/fixtures/apps/watcher-type-default/config/config.unittest.js
index bef652008..a3ba4c490 100644
--- a/test/fixtures/apps/watcher-type-default/config/config.unittest.js
+++ b/test/fixtures/apps/watcher-type-default/config/config.unittest.js
@@ -3,3 +3,5 @@
 exports.watcher = {
   type: 'default',
 };
+
+exports.keys = 'test key';
diff --git a/test/fixtures/apps/worker-die/config/config.default.js b/test/fixtures/apps/worker-die/config/config.default.js
new file mode 100644
index 000000000..1c7cb2ea0
--- /dev/null
+++ b/test/fixtures/apps/worker-die/config/config.default.js
@@ -0,0 +1,3 @@
+'use strict';
+
+exports.keys = 'test key';
diff --git a/test/lib/application.test.js b/test/lib/application.test.js
index bafe00b80..4cc690261 100644
--- a/test/lib/application.test.js
+++ b/test/lib/application.test.js
@@ -53,6 +53,7 @@ describe('test/lib/application.test.js', () => {
       mm.env('test');
       app = utils.app('apps/keys-missing');
       yield app.ready();
+      mm(app.config, 'keys', null);
 
       try {
         app.keys;
@@ -65,15 +66,37 @@ describe('test/lib/application.test.js', () => {
       yield app.close();
     });
 
-    it('should auto set keys on unittest', function* () {
+    it('should throw when config.keys missing on unittest env', function* () {
       mm.env('unittest');
       app = utils.app('apps/keys-missing');
       yield app.ready();
+      mm(app.config, 'keys', null);
 
-      assert(app.keys);
-      assert(app.keys);
-      assert(app.config.keys === 'foo, keys, you need to set your app keys');
+      try {
+        app.keys;
+        throw new Error('should not run this');
+      } catch (err) {
+        assert(err.message === 'Please set config.keys first');
+      }
 
+      // make sure app close
+      yield app.close();
+    });
+
+    it('should throw when config.keys missing on local env', function* () {
+      mm.env('local');
+      app = utils.app('apps/keys-missing');
+      yield app.ready();
+      mm(app.config, 'keys', null);
+
+      try {
+        app.keys;
+        throw new Error('should not run this');
+      } catch (err) {
+        assert(err.message === 'Please set config.keys first');
+      }
+
+      // make sure app close
       yield app.close();
     });
 
diff --git a/test/utils.js b/test/utils.js
index cfdf7f170..10d45958b 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -9,7 +9,8 @@ const eggPath = path.join(__dirname, '..');
 
 exports.app = (name, options) => {
   options = formatOptions(name, options);
-  return mm.app(options);
+  const app = mm.app(options);
+  return app;
 };
 
 /**