Refactor of codes in node.js

1) Add async method: `binarySearch` and `btreeSearch`.
2) Refactor of tests by putting them into `tests` folder.
3) Create several new tests in details.
4) Remove useless and reformat codes to be clear.
5) Remove useless snapshots, because they can be recreated when you run `npm run test`.
This commit is contained in:
MaleDong 2018-07-15 11:08:23 +08:00
parent 0a7b2e225f
commit 35fc3cbf23
11 changed files with 644 additions and 161 deletions

View File

@ -1,3 +1,56 @@
# nodejs 客户端
现已实现同步查询,异步查询未实现,用法参考 ip2region.spec.js
## 实现情况:
现已实现同步和异步查询,具体使用方法可以参考 `nodejs\tests\constructorTest.spec.js``nodejs\tests\createTest.spec.js`
## 如何贡献?
你可以任意修改代码,但必须确保通过全部的单元测试。要保证通过全部的单元测试,请在 Nodejs 控制台下切换到 nodejs 目录:
1在此之前请先运行 `npm i` 确保你已经安装了各类初始化第三方工具。
2然后运行 `npm run coverage` 确保你的代码可以通过全部测试必要时可以添加测试同时保证代码覆盖率都是绿色80%以上)。
```bash
D:\Projects\ip2region\binding\nodejs>npm run coverage
> ip2region@0.0.1 coverage D:\Projects\ip2region\binding\nodejs
> npm run test && jest --coverage
> ip2region@0.0.1 test D:\Projects\ip2region\binding\nodejs
> jest
PASS tests\constructorTest.spec.js
PASS tests\createTest.spec.js
PASS tests\exceptionTest.spec.js
Snapshot Summary
168 snapshots written in 2 test suites.
Test Suites: 3 passed, 3 total
Tests: 14 passed, 14 total
Snapshots: 168 added, 168 total
Time: 1.645s
Ran all test suites.
PASS tests\constructorTest.spec.js
PASS tests\createTest.spec.js
PASS tests\exceptionTest.spec.js
----------------------|----------|----------|----------|----------|-------------------|
File | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s |
----------------------|----------|----------|----------|----------|-------------------|
All files | 92.34 | 80.77 | 96 | 93.83 | |
nodejs | 91.95 | 80.26 | 95.65 | 93.51 | |
ip2region.js | 91.95 | 80.26 | 95.65 | 93.51 |... 09,410,460,484 |
nodejs/tests/utils | 100 | 100 | 100 | 100 | |
asyncFor.js | 100 | 100 | 100 | 100 | |
fetchMainVersion.js | 100 | 100 | 100 | 100 | |
testData.js | 100 | 100 | 100 | 100 | |
----------------------|----------|----------|----------|----------|-------------------|
Test Suites: 3 passed, 3 total
Tests: 14 passed, 14 total
Snapshots: 168 passed, 168 total
Time: 1.792s
Ran all test suites.
```

View File

@ -1,29 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`ip2region binarySearch 1`] = `
Object {
"city": 2163,
"region": "中国|0|广东省|深圳市|阿里云",
}
`;
exports[`ip2region binarySearch 2`] = `
Object {
"city": 0,
"region": "0|0|0|内网IP|内网IP",
}
`;
exports[`ip2region should query 1`] = `
Object {
"city": 2163,
"region": "中国|0|广东省|深圳市|阿里云",
}
`;
exports[`ip2region should query 2`] = `
Object {
"city": 0,
"region": "0|0|0|内网IP|内网IP",
}
`;

View File

@ -5,20 +5,18 @@
*
* @author dongyado<dongyado@gmail.com>
* @author leeching<leeching.fx@gmail.com>
* @author dongwei<maledong_github@outlook.com>
*/
const fs = require('fs');
const IP_BASE = [16777216, 65536, 256, 1];
const INDEX_BLOCK_LENGTH = 12;
const TOTAL_HEADER_LENGTH = 8192;
//#region Private Functions
/**
* Convert ip to long (xxx.xxx.xxx.xxx to a integer)
*
* @param {string} ip
* @return {number} long value
*/
function ip2long(ip) {
function _ip2long(ip) {
const arr = ip.split('.');
if (arr.length !== 4) {
throw new Error('invalid ip');
@ -39,7 +37,7 @@ function ip2long(ip) {
* @param {number} offset
* @return {number} long value
*/
function getLong(buffer, offset) {
function _getLong(buffer, offset) {
const val =
(buffer[offset] & 0x000000ff) |
((buffer[offset + 1] << 8) & 0x0000ff00) |
@ -48,82 +46,156 @@ function getLong(buffer, offset) {
return val < 0 ? val >>> 0 : val;
}
/**
* @typedef {Object} SearchResult
* @property {number} city
* @property {string} region
*/
//#endregion
//#region Private Variables
// We don't wanna expose a private global settings to
// the public for safety reason.
const _globalInstances = new Map();
const IP_BASE = [16777216, 65536, 256, 1];
const INDEX_BLOCK_LENGTH = 12;
const TOTAL_HEADER_LENGTH = 8192;
// Private Message Symbols for functions
const PrepareHeader = Symbol('#PrepareHeader');
const CalTotalBlocks = Symbol('#CalsTotalBlocks');
const ReadDataSync = Symbol('#ReadDataSync');
const ReadData = Symbol('#ReadData');
//#endregion
class IP2Region {
static create(dbPath) {
const oldInstance = IP2Region._instances.get(dbPath);
if (oldInstance) {
return oldInstance;
} else {
const instance = new IP2Region({ dbPath });
IP2Region._instances.set(dbPath, instance);
return instance;
//#region Private Functions
[CalTotalBlocks]() {
const superBlock = new Buffer(8);
fs.readSync(this.dbFd, superBlock, 0, 8, 0);
this.firstIndexPtr = _getLong(superBlock, 0);
this.lastIndexPtr = _getLong(superBlock, 4);
this.totalBlocks =
(this.lastIndexPtr - this.firstIndexPtr) / INDEX_BLOCK_LENGTH + 1;
}
[PrepareHeader]() {
fs.readSync(
this.dbFd,
this.headerIndexBuffer,
0,
TOTAL_HEADER_LENGTH,
8
);
for (let i = 0; i < TOTAL_HEADER_LENGTH; i += 8) {
const startIp = _getLong(this.headerIndexBuffer, i);
const dataPtr = _getLong(this.headerIndexBuffer, i + 4);
if (dataPtr == 0) break;
this.headerSip.push(startIp);
this.headerPtr.push(dataPtr);
this.headerLen++; // header index size count
}
}
[ReadData](dataPos, callBack) {
if (dataPos == 0) return callBack(null,null);
const dataLen = (dataPos >> 24) & 0xff;
dataPos = dataPos & 0x00ffffff;
const dataBuffer = new Buffer(dataLen);
fs.read(this.dbFd, dataBuffer, 0, dataLen, dataPos, (err, result) => {
if (err) {
callBack(err, null);
}
else {
const city = _getLong(dataBuffer, 0);
const region = dataBuffer.toString('utf8', 4, dataLen);
callBack(null, { city, region });
}
});
}
[ReadDataSync](dataPos) {
if (dataPos == 0) return null;
const dataLen = (dataPos >> 24) & 0xff;
dataPos = dataPos & 0x00ffffff;
const dataBuffer = new Buffer(dataLen);
fs.readSync(this.dbFd, dataBuffer, 0, dataLen, dataPos);
const city = _getLong(dataBuffer, 0);
const region = dataBuffer.toString('utf8', 4, dataLen);
return { city, region };
}
//#endregion
//#region Static Functions
// Single Instance
static create(dbPath) {
let existInstance = _globalInstances.get(dbPath);
if (existInstance == null) {
existInstance = new IP2Region({ dbPath: dbPath });
}
return existInstance;
}
/**
* For backward compatibility
*/
static destroy() {
IP2Region._instances.forEach(([key, instance]) => {
_globalInstances.forEach(([key, instance]) => {
instance.destroy();
});
}
//#endregion
constructor(options = {}) {
const { dbPath } = options;
if (!dbPath || !fs.existsSync(dbPath)) {
throw new Error(`[ip2region] db file not exists : ${dbPath}`);
}
try {
this.dbFd = fs.openSync(dbPath, 'r');
} catch (e) {
throw new Error(
`[ip2region] Can not open ip2region.db file , path: ${dbPath}`
);
}
this.dbFd = fs.openSync(dbPath, 'r');
IP2Region._instances.set((this.dbPath = dbPath), this);
this.dbPath = dbPath;
_globalInstances.set(this.dbPath, this);
this.totalBlocks = this.firstIndexPtr = this.lastIndexPtr = 0;
this.calcTotalBlocks();
this[CalTotalBlocks]();
this.headerIndexBuffer = new Buffer(TOTAL_HEADER_LENGTH);
this.headerSip = [];
this.headerPtr = [];
this.headerLen = 0;
this.prepareHeader();
this[PrepareHeader]();
}
//#region Public Functions
/**
* @public
* Destroy the current file by closing it.
*/
destroy() {
fs.closeSync(ip2rObj.dbFd);
IP2Region._instances.delete(this.dbPath);
fs.closeSync(this.dbFd);
_globalInstances.delete(this.dbPath);
}
/**
* @public
* @param {string} ip
* @return {SearchResult}
* Sync of binarySearch.
* @param {string} ip The IP address to search for.
* @return {SearchResult} A result something like `{ city: 2163, region: '中国|0|广东省|深圳市|阿里云' }`
*/
binarySearchSync(ip) {
ip = ip2long(ip);
ip = _ip2long(ip);
let low = 0;
let mid = 0;
let high = this.totalBlocks;
let dataPos = 0;
let pos = 0;
let sip = 0;
let eip = 0;
const indexBuffer = new Buffer(12);
// binary search
@ -131,30 +203,89 @@ class IP2Region {
mid = (low + high) >> 1;
pos = this.firstIndexPtr + mid * INDEX_BLOCK_LENGTH;
fs.readSync(this.dbFd, indexBuffer, 0, INDEX_BLOCK_LENGTH, pos);
sip = getLong(indexBuffer, 0);
sip = _getLong(indexBuffer, 0);
if (ip < sip) {
high = mid - 1;
} else {
eip = getLong(indexBuffer, 4);
if (ip > eip) {
sip = _getLong(indexBuffer, 4);
if (ip > sip) {
low = mid + 1;
} else {
dataPos = getLong(indexBuffer, 8);
sip = _getLong(indexBuffer, 8);
break;
}
}
}
return this.readData(dataPos);
return this[ReadDataSync](sip);
}
/**
* @public
* @param {string} ip
* @return {SearchResult}
* Async of binarySearch.
* @param {string} ip The IP address to search for.
* @param {Function} callBack The callBack function with two parameters, if successful,
* err is null and result is `{ city: 2163, region: '中国|0|广东省|深圳市|阿里云' }`
*/
binarySearch(ip, callBack) {
ip = _ip2long(ip);
let low = 0;
let mid = 0;
let high = this.totalBlocks;
let pos = 0;
let sip = 0;
const indexBuffer = new Buffer(12);
const _self = this;
// Because `while` is a sync method, we have to convert this to a recursive loop
// and in each loop we should continue calling `setImmediate` until we found the IP.
function _innerAsyncWhile() {
if (low <= high) {
mid = (low + high) >> 1;
pos = _self.firstIndexPtr + mid * INDEX_BLOCK_LENGTH;
// Now async read the file
fs.read(_self.dbFd, indexBuffer, 0, INDEX_BLOCK_LENGTH, pos, (err) => {
if (err) {
return callBack(err, null);
}
sip = _getLong(indexBuffer, 0);
if (ip < sip) {
high = mid - 1;
setImmediate(_innerAsyncWhile);
} else {
sip = _getLong(indexBuffer, 4);
if (ip > sip) {
low = mid + 1;
setImmediate(_innerAsyncWhile);
} else {
sip = _getLong(indexBuffer, 8);
_self[ReadData](sip, (err, result) => {
callBack(err, result);
});
}
}
});
}
}
// Call this immediately
_innerAsyncWhile();
}
/**
* Sync of btreeSearch.
* @param {string} ip The IP address to search for.
* @return {Function} A result something like `{ city: 2163, region: '中国|0|广东省|深圳市|阿里云' }`
*/
btreeSearchSync(ip) {
ip = ip2long(ip);
ip = _ip2long(ip);
// first search (in header index)
let low = 0;
@ -221,84 +352,143 @@ class IP2Region {
let p = 0;
let sip = 0;
let eip = 0;
let dataPtr = 0;
while (low <= high) {
mid = (low + high) >> 1;
p = mid * INDEX_BLOCK_LENGTH;
sip = getLong(blockBuffer, p);
sip = _getLong(blockBuffer, p);
if (ip < sip) {
high = mid - 1;
} else {
eip = getLong(blockBuffer, p + 4);
if (ip > eip) {
sip = _getLong(blockBuffer, p + 4);
if (ip > sip) {
low = mid + 1;
} else {
dataPtr = getLong(blockBuffer, p + 8);
sip = _getLong(blockBuffer, p + 8);
break;
}
}
}
return this.readData(dataPtr);
return this[ReadDataSync](sip);
}
/**
* @private
*/
calcTotalBlocks() {
const superBlock = new Buffer(8);
fs.readSync(this.dbFd, superBlock, 0, 8, 0);
this.firstIndexPtr = getLong(superBlock, 0);
this.lastIndexPtr = getLong(superBlock, 4);
this.totalBlocks =
(this.lastIndexPtr - this.firstIndexPtr) / INDEX_BLOCK_LENGTH + 1;
}
* Async of btreeSearch.
* @param {string} ip The IP address to search for.
* @param {Function} callBack The callBack function with two parameters, if successful,
* err is null and result is `{ city: 2163, region: '中国|0|广东省|深圳市|阿里云' }`
*/
btreeSearch(ip, callBack) {
ip = _ip2long(ip);
/**
* @private
*/
prepareHeader() {
fs.readSync(
this.dbFd,
this.headerIndexBuffer,
0,
TOTAL_HEADER_LENGTH,
8
);
// first search (in header index)
let low = 0;
let mid = 0;
let high = this.headerLen;
let sptr = 0;
let eptr = 0;
for (let i = 0; i < TOTAL_HEADER_LENGTH; i += 8) {
const startIp = getLong(this.headerIndexBuffer, i);
const dataPtr = getLong(this.headerIndexBuffer, i + 4);
if (dataPtr == 0) break;
while (low <= high) {
mid = (low + high) >> 1;
this.headerSip.push(startIp);
this.headerPtr.push(dataPtr);
this.headerLen++; // header index size count
if (ip == this.headerSip[mid]) {
if (mid > 0) {
sptr = this.headerPtr[mid - 1];
eptr = this.headerPtr[mid];
} else {
sptr = this.headerPtr[mid];
eptr = this.headerPtr[mid + 1];
}
break;
}
if (ip < this.headerSip[mid]) {
if (mid == 0) {
sptr = this.headerPtr[mid];
eptr = this.headerPtr[mid + 1];
break;
} else if (ip > this.headerSip[mid - 1]) {
sptr = this.headerPtr[mid - 1];
eptr = this.headerPtr[mid];
break;
}
high = mid - 1;
} else {
if (mid == this.headerLen - 1) {
sptr = this.headerPtr[mid - 1];
eptr = this.headerPtr[mid];
break;
} else if (ip <= this.headerSip[mid + 1]) {
sptr = this.headerPtr[mid];
eptr = this.headerPtr[mid + 1];
break;
}
low = mid + 1;
}
}
// match nothing
if (sptr == 0) return callBack(null, null);
let p = 0;
let sip = 0;
// second search (in index)
const blockLen = eptr - sptr;
const blockBuffer = new Buffer(blockLen + INDEX_BLOCK_LENGTH);
low = 0;
high = blockLen / INDEX_BLOCK_LENGTH;
const _self = this;
function _innerAsyncWhile() {
if (low <= high) {
mid = (low + high) >> 1;
p = mid * INDEX_BLOCK_LENGTH;
// Use this to call the method itself as
// an asynchronize step
fs.read(_self.dbFd, blockBuffer,
0,
blockLen + INDEX_BLOCK_LENGTH,
sptr, (err) => {
if (err) {
return callBack(err, null);
}
sip = _getLong(blockBuffer, p);
if (ip < sip) {
high = mid - 1;
setImmediate(_innerAsyncWhile);
} else {
sip = _getLong(blockBuffer, p + 4);
if (ip > sip) {
low = mid + 1;
setImmediate(_innerAsyncWhile);
} else {
sip = _getLong(blockBuffer, p + 8);
_self[ReadData](sip, (err, result) => {
callBack(err, result);
});
}
}
});
}
else {
// If we found nothing, return null
return callBack(null, null);
}
}
_innerAsyncWhile();
}
/**
* @private
* @param {number} dataPos
* @return {SearchResult}
*/
readData(dataPos) {
if (dataPos == 0) return null;
const dataLen = (dataPos >> 24) & 0xff;
dataPos = dataPos & 0x00ffffff;
const dataBuffer = new Buffer(dataLen);
fs.readSync(this.dbFd, dataBuffer, 0, dataLen, dataPos);
const city = getLong(dataBuffer, 0);
const region = dataBuffer.toString('utf8', 4, dataLen);
return { city, region };
}
//#endregion
}
IP2Region._instances = new Map();
module.exports = IP2Region;

View File

@ -1,24 +0,0 @@
const path = require('path');
const IP2Region = require('./ip2region');
describe('ip2region', () => {
let instance;
beforeAll(() => {
instance = IP2Region.create(path.join(__dirname, '../../data/ip2region.db'));
});
afterAll(() => {
instance.destroy();
});
test('should query', () => {
expect(instance.btreeSearchSync('120.24.78.68')).toMatchSnapshot();
expect(instance.btreeSearchSync('10.10.10.10')).toMatchSnapshot();
});
test('binarySearch', () => {
expect(instance.binarySearchSync('120.24.78.68')).toMatchSnapshot();
expect(instance.binarySearchSync('10.10.10.10')).toMatchSnapshot();
});
});

View File

@ -4,7 +4,8 @@
"description": "ip database ",
"main": "ip2region.js",
"scripts": {
"test": "jest"
"test": "jest",
"coverage":"npm run test && jest --coverage"
},
"repository": {
"type": "git",

View File

@ -0,0 +1,108 @@
// This test is used for tesing of a static function `create` of IP2Region
const IP2Region = require('../ip2region');
const testIps = require('./utils/testData');
const asyncFor = require('./utils/asyncFor');
describe('Constructor Test', () => {
let instance;
beforeAll(() => {
instance = new IP2Region({ dbPath: '../../data/ip2region.db' });
});
afterAll(() => {
IP2Region.destroy();
});
test('btreeSearchSync query', () => {
for (const ip of testIps) {
expect(instance.btreeSearchSync(ip)).toMatchSnapshot();
}
});
test('binarySearchSync query', () => {
for (const ip of testIps) {
expect(instance.binarySearchSync(ip)).toMatchSnapshot();
}
});
//#region callBack
test('binarySearch query', (done) => {
asyncFor(testIps,
(value, continueCallBack) => {
instance.binarySearch(value, (err, result) => {
expect(err).toBe(null);
expect(result).toMatchSnapshot();
continueCallBack();
});
},
() => { done() });
});
test('btreeSearch query', (done) => {
asyncFor(testIps,
(value, continueCallBack) => {
instance.btreeSearch(value, (err, result) => {
expect(err).toBe(null);
expect(result).toMatchSnapshot();
continueCallBack();
});
},
() => { done() });
});
//#endregion
//#region Async Promisify test
const node_ver = require('./utils/fetchMainVersion');
// If we have Nodejs >= 8, we now support `async` and `await`
if (node_ver >= 8) {
const asyncBinarySearch = async (ip) => {
return new Promise((resolve, reject) => {
instance.binarySearch(ip, (err, result) => {
if (err) {
reject(err);
}
else {
resolve(result);
}
});
});
};
const asyncBtreeSearch = async (ip) => {
return new Promise((resolve, reject) => {
instance.btreeSearch(ip, (err, result) => {
if (err) {
reject(err);
}
else {
resolve(result);
}
});
});
};
test('async binarySearch query', async () => {
for (let i = 0; i < testIps.length; ++i) {
const result = await asyncBinarySearch(testIps[i]);
expect(result).toMatchSnapshot();
}
});
test('async btreeSearch query', async () => {
for (let i = 0; i < testIps.length; ++i) {
const result = await asyncBtreeSearch(testIps[i]);
expect(result).toMatchSnapshot();
}
});
}
//#endregion
});

View File

@ -0,0 +1,108 @@
// This test is used for tesing of a static function `create` of IP2Region
const IP2Region = require('../ip2region');
const testIps = require('./utils/testData');
const asyncFor = require('./utils/asyncFor');
describe('Create Test', () => {
let instance;
beforeAll(() => {
instance = IP2Region.create('../../data/ip2region.db');
});
afterAll(() => {
IP2Region.destroy();
});
test('btreeSearchSync query', () => {
for (const ip of testIps) {
expect(instance.btreeSearchSync(ip)).toMatchSnapshot();
}
});
test('binarySearchSync query', () => {
for (const ip of testIps) {
expect(instance.binarySearchSync(ip)).toMatchSnapshot();
}
});
//#region callBack
test('binarySearch query', (done) => {
asyncFor(testIps,
(value, continueCallBack) => {
instance.binarySearch(value, (err, result) => {
expect(err).toBe(null);
expect(result).toMatchSnapshot();
continueCallBack();
});
},
() => { done() });
});
test('btreeSearch query', (done) => {
asyncFor(testIps,
(value, continueCallBack) => {
instance.btreeSearch(value, (err, result) => {
expect(err).toBe(null);
expect(result).toMatchSnapshot();
continueCallBack();
});
},
() => { done() });
});
//#endregion
//#region Async Promisify test
const node_ver = require('./utils/fetchMainVersion');
// If we have Nodejs >= 8, we now support `async` and `await`
if (node_ver >= 8) {
const asyncBinarySearch = async (ip) => {
return new Promise((resolve, reject) => {
instance.binarySearch(ip, (err, result) => {
if (err) {
reject(err);
}
else {
resolve(result);
}
});
});
};
const asyncBtreeSearch = async (ip) => {
return new Promise((resolve, reject) => {
instance.btreeSearch(ip, (err, result) => {
if (err) {
reject(err);
}
else {
resolve(result);
}
});
});
};
test('async binarySearch query', async () => {
for (let i = 0; i < testIps.length; ++i) {
const result = await asyncBinarySearch(testIps[i]);
expect(result).toMatchSnapshot();
}
});
test('async btreeSearch query', async () => {
for (let i = 0; i < testIps.length; ++i) {
const result = await asyncBtreeSearch(testIps[i]);
expect(result).toMatchSnapshot();
}
});
}
//#endregion
});

View File

@ -0,0 +1,28 @@
// This test is used for tesing of exceptions
const IP2Region = require('../ip2region');
describe('Constructor Test', () => {
let instance;
beforeAll(() => {
instance = new IP2Region({ dbPath: '../../data/ip2region.db' })
});
afterAll(() => {
instance.destroy();
});
test('IP invalid test', () => {
const invalidIps = ['255.234.233', '255.255.-1.255', null, undefined, '', 'x.255.y.200'];
for (const ip of invalidIps) {
expect(() => instance.btreeSearchSync(ip)).toThrow();
expect(() => instance.binarySearchSync(ip)).toThrow();
}
});
test('File Not Found test', () => {
expect(() => new IP2Region({ dbPath: 'A Bad File or Path Here' })).toThrow();
});
});

View File

@ -0,0 +1,23 @@
/**
* Async For
* @param {Array} groupArray
* @param {Function} exeCallBack
* @param {Function} finalCallBack
*/
function asyncFor(groupArray, exeCallBack, finalCallBack) {
let i = 0;
function _innerAsyncLoop() {
if (i < groupArray.length) {
exeCallBack(groupArray[i++], _innerAsyncLoop);
}
else {
finalCallBack();
}
}
_innerAsyncLoop();
}
module.exports = asyncFor;

View File

@ -0,0 +1,9 @@
let node_ver = process.version
// Because nodejs's version is something like `v8.11.3`. So we should ignore `v` first
node_ver = node_ver.substr(1);
// Splitted by `.`
node_ver = node_ver.split('.');
// Take the main version number
node_ver = parseInt(node_ver[0]);
module.exports = node_ver;

View File

@ -0,0 +1,16 @@
module.exports = [
'0.0.0.0',
'10.10.10.10',
'210.109.255.230',
'192.168.0.1',
'255.255.255.255',
'77.49.66.88',
'210.248.255.231',
'35.193.251.120',
'197.84.60.202',
'183.196.233.159',
'20.108.91.101',
'120.196.148.137',
'249.255.250.200',
'112.65.1.130'
]