archive-gh-me/ip2region
1
0
Fork 0
mirror of https://github.com/lionsoul2014/ip2region.git synced 2025-12-08 19:25:22 +00:00
Code Issues Projects Releases Wiki Activity

[Feature] Add 'memorySearch' and 'memorySearchSync' for Nodejs

Browse Source 
1) Add features.
2) Add unit tests.
3) Add benchmark.
This commit is contained in:
MaleDong 2018-07-31 15:48:05 +08:00
parent f4fba7db60
commit b9e9fcfd80
9 changed files with 4402 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -37,3 +37,7 @@ target
/binding/c#/**/packages
/binding/c#/**/bin
/binding/c#/**/obj
# Nodejs
/binding/nodejs/tests/unitTests/__snapshots__
/binding/nodejs/coverage

19
binding/nodejs/README.md
View File

@ -9,7 +9,7 @@
你可以任意修改代码，但必须确保通过全部的单元测试。要保证通过全部的单元测试，请在 Nodejs 控制台下切换到 nodejs 目录：
1）在此之前，请先运行 `npm i` 确保你已经安装了各类初始化第三方工具。
2）然后运行 `npm run coverage` 确保你的代码可以通过全部测试（必要时可以添加测试），同时保证代码覆盖率都是绿色（80%以上）。
2）然后运行 `npm run coverage` 确保你的代码可以通过全部测试（必要时可以添加测试）。
```bash
D:\Projects\ip2region\binding\nodejs>npm run coverage
@ -53,4 +53,21 @@ Tests: 14 passed, 14 total
Snapshots: 168 passed, 168 total
Time: 1.792s
Ran all test suites.
```
3）使用benchmark测试，结果如下：
```bash
D:\Projects\ip2region\binding\nodejs>node D:\Projects\ip2region\binding\nodejs\tests\benchmarkTests\main.js
MemorySearchSync x 55,969 ops/sec ±2.22% (90 runs sampled)
BinarySearchSync x 610 ops/sec ±5.41% (77 runs sampled)
BtreeSearchSync x 2,439 ops/sec ±6.93% (69 runs sampled)
MemorySearch x 2,924 ops/sec ±0.67% (85 runs sampled)
BinarySearch x 154 ops/sec ±2.20% (69 runs sampled)
BtreeSearch x 294 ops/sec ±2.58% (76 runs sampled)
Rand Name Time (in milliseconds)
1 MemorySearchSync 0.018
2 MemorySearch 0.342
3 BtreeSearchSync 0.410
4 BinarySearchSync 1.639
5 BtreeSearch 3.407
6 BinarySearch 6.497
```

162
binding/nodejs/ip2region.js
View File

@ -70,7 +70,7 @@ class IP2Region {
//#region Private Functions
[CalTotalBlocks]() {
const superBlock = new Buffer(8);
const superBlock = Buffer.alloc(8);
fs.readSync(this.dbFd, superBlock, 0, 8, 0);
this.firstIndexPtr = _getLong(superBlock, 0);
this.lastIndexPtr = _getLong(superBlock, 4);
@ -99,10 +99,10 @@ class IP2Region {
}
[ReadData](dataPos, callBack) {
if (dataPos == 0) return callBack(null,null);
if (dataPos == 0) return callBack(null, null);
const dataLen = (dataPos >> 24) & 0xff;
dataPos = dataPos & 0x00ffffff;
const dataBuffer = new Buffer(dataLen);
const dataBuffer = Buffer.alloc(dataLen);
fs.read(this.dbFd, dataBuffer, 0, dataLen, dataPos, (err, result) => {
if (err) {
@ -120,7 +120,7 @@ class IP2Region {
if (dataPos == 0) return null;
const dataLen = (dataPos >> 24) & 0xff;
dataPos = dataPos & 0x00ffffff;
const dataBuffer = new Buffer(dataLen);
const dataBuffer = Buffer.alloc(dataLen);
fs.readSync(this.dbFd, dataBuffer, 0, dataLen, dataPos);
@ -157,6 +157,10 @@ class IP2Region {
const { dbPath } = options;
// Keep for MemorySearch
this.totalInMemoryBytesSize = fs.statSync(dbPath).size;
this.totalInMemoryBytes = null;
this.dbFd = fs.openSync(dbPath, 'r');
this.dbPath = dbPath;
@ -166,7 +170,7 @@ class IP2Region {
this.totalBlocks = this.firstIndexPtr = this.lastIndexPtr = 0;
this[CalTotalBlocks]();
this.headerIndexBuffer = new Buffer(TOTAL_HEADER_LENGTH);
this.headerIndexBuffer = Buffer.alloc(TOTAL_HEADER_LENGTH);
this.headerSip = [];
this.headerPtr = [];
this.headerLen = 0;
@ -196,7 +200,7 @@ class IP2Region {
let high = this.totalBlocks;
let pos = 0;
let sip = 0;
const indexBuffer = new Buffer(12);
const indexBuffer = Buffer.alloc(12);
// binary search
while (low <= high) {
@ -235,7 +239,7 @@ class IP2Region {
let high = this.totalBlocks;
let pos = 0;
let sip = 0;
const indexBuffer = new Buffer(12);
const indexBuffer = Buffer.alloc(12);
const _self = this;
// Because `while` is a sync method, we have to convert this to a recursive loop
@ -338,7 +342,7 @@ class IP2Region {
// second search (in index)
const blockLen = eptr - sptr;
const blockBuffer = new Buffer(blockLen + INDEX_BLOCK_LENGTH);
const blockBuffer = Buffer.alloc(blockLen + INDEX_BLOCK_LENGTH);
fs.readSync(
this.dbFd,
blockBuffer,
@ -436,7 +440,7 @@ class IP2Region {
// second search (in index)
const blockLen = eptr - sptr;
const blockBuffer = new Buffer(blockLen + INDEX_BLOCK_LENGTH);
const blockBuffer = Buffer.alloc(blockLen + INDEX_BLOCK_LENGTH);
low = 0;
high = blockLen / INDEX_BLOCK_LENGTH;
@ -488,7 +492,141 @@ class IP2Region {
_innerAsyncWhile();
}
//#endregion
}
/**
* Sync of MemorySearch.
* @param {String} ip
*/
memorySearchSync(ip) {
module.exports = IP2Region;
ip = _ip2long(ip);
if (this.totalInMemoryBytes === null) {
this.totalInMemoryBytes = Buffer.alloc(this.totalInMemoryBytesSize);
fs.readSync(this.dbFd, this.totalInMemoryBytes, 0, this.totalInMemoryBytesSize, 0);
this.firstIndexPtr = _getLong(this.totalInMemoryBytes, 0);
this.lastIndexPtr = _getLong(this.totalInMemoryBytes, 4);
this.totalBlocks = ((this.lastIndexPtr - this.firstIndexPtr) / INDEX_BLOCK_LENGTH) | 0 + 1;
}
let l = 0, h = this.totalBlocks;
let sip = 0;
let m = 0, p = 0;
while (l <= h) {
m = (l + h) >> 1;
p = (this.firstIndexPtr + m * INDEX_BLOCK_LENGTH) | 0;
sip = _getLong(this.totalInMemoryBytes, p);
if (ip < sip) {
h = m - 1;
}
else {
sip = _getLong(this.totalInMemoryBytes, p + 4);
if (ip > sip) {
l = m + 1;
}
else {
sip = _getLong(this.totalInMemoryBytes, p + 8);
//not matched
if (sip === 0) return null;
//get the data
let dataLen = ((sip >> 24) & 0xFF) | 0;
let dataPtr = ((sip & 0x00FFFFFF)) | 0;
let city = _getLong(this.totalInMemoryBytes, dataPtr);
const bufArray = new Array();
for (let startPos = dataPtr + 4, i = startPos; i < startPos + dataLen - 4; ++i) {
bufArray.push(this.totalInMemoryBytes[i]);
}
const region = Buffer.from(bufArray, 0).toString();
return { city, region };
}
}
}
}
/**
* Async of MemorySearch.
* @param {String} ip
*/
memorySearch(ip, callBack) {
let _ip = _ip2long(ip);
let l = 0, h = this.totalBlocks;
let sip = 0;
let m = 0, p = 0;
let self = this;
function _innerMemorySearchLoop() {
if (l <= h) {
m = (l + h) >> 1;
p = (self.firstIndexPtr + m * INDEX_BLOCK_LENGTH) | 0;
sip = _getLong(self.totalInMemoryBytes, p);
if (_ip < sip) {
h = m - 1;
setImmediate(_innerMemorySearchLoop);
}
else {
sip = _getLong(self.totalInMemoryBytes, p + 4);
if (_ip > sip) {
l = m + 1;
setImmediate(_innerMemorySearchLoop);
}
else {
sip = _getLong(self.totalInMemoryBytes, p + 8);
//not matched
if (sip === 0) return callBack(null, null);
//get the data
let dataLen = ((sip >> 24) & 0xFF) | 0;
let dataPtr = ((sip & 0x00FFFFFF)) | 0;
let city = _getLong(self.totalInMemoryBytes, dataPtr);
const bufArray = new Array();
for (let startPos = dataPtr + 4, i = startPos; i < startPos + dataLen - 4; ++i) {
bufArray.push(self.totalInMemoryBytes[i]);
}
const region = Buffer.from(bufArray).toString();
callBack(null, { city, region });
}
}
}
else {
callBack(null, null);
}
}
if (this.totalInMemoryBytes === null) {
this.totalInMemoryBytes = Buffer.alloc(this.totalInMemoryBytesSize);
fs.read(this.dbFd, this.totalInMemoryBytes, 0, this.totalInMemoryBytesSize, 0, (err) => {
if (err) {
callBack(err, null);
}
else {
this.firstIndexPtr = _getLong(this.totalInMemoryBytes, 0);
this.lastIndexPtr = _getLong(this.totalInMemoryBytes, 4);
this.totalBlocks = ((this.lastIndexPtr - this.firstIndexPtr) / INDEX_BLOCK_LENGTH) | 0 + 1;
_innerMemorySearchLoop();
}
});
}
else {
_innerMemorySearchLoop();
}
}
}
//#endregion
module.exports = IP2Region;

4049
binding/nodejs/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

5
binding/nodejs/package.json
View File

@ -5,7 +5,7 @@
"main": "ip2region.js",
"scripts": {
"test": "jest",
"coverage":"npm run test && jest --coverage"
"coverage": "npm run test && jest --coverage"
},
"repository": {
"type": "git",
@ -23,6 +23,7 @@
},
"homepage": "https://github.com/lionsoul2016/ip2region",
"devDependencies": {
"jest": "^19.0.2"
"jest": "^19.0.2",
"benchmark": "^2.1.4"
}
}

94
binding/nodejs/tests/benchmarkTests/main.js Normal file
View File

@ -0,0 +1,94 @@
const Benchmark = require('benchmark');
const suite = new Benchmark.Suite();
const searcher = require('../../ip2region').create('../../data/ip2region.db');
const testDatas = require('../utils/testData');
const asyncFor = require('../utils/asyncFor');
suite.add("MemorySearchSync", () => {
for (let i = 0; i < testDatas.length; ++i) {
searcher.memorySearchSync(testDatas[i]);
}
})
.add("BinarySearchSync", () => {
for (let i = 0; i < testDatas.length; ++i) {
searcher.binarySearchSync(testDatas[i]);
}
})
.add("BtreeSearchSync", () => {
for (let i = 0; i < testDatas.length; ++i) {
searcher.btreeSearchSync(testDatas[i]);
}
})
.add("MemorySearch", {
defer: true,
fn: function (completeCallBack) {
asyncFor(testDatas,
(v, c) => {
searcher.memorySearch(v, () => {
c();
});
},
() => {
completeCallBack.resolve();
});
}
})
.add("BinarySearch", {
defer: true,
fn: function (completeCallBack) {
asyncFor(testDatas,
(v, c) => {
searcher.binarySearch(v, () => {
c();
});
},
() => {
completeCallBack.resolve();
});
}
})
.add("BtreeSearch", {
defer: true,
fn: function (completeCallBack) {
asyncFor(testDatas,
(v, c) => {
searcher.btreeSearch(v, () => {
c();
});
},
() => {
completeCallBack.resolve();
});
}
})
.on('cycle', function (event) {
console.log(String(event.target));
})
.on('complete', function () {
let results = new Array();
for (let prop in this) {
if (!isNaN(prop)) {
const eachResult = {
name: this[prop].name,
mean: this[prop].stats.mean * 1000, //second => millisecond
moe: this[prop].stats.moe,
rme: this[prop].stats.rme,
sem: this[prop].stats.sem
}
results.push(eachResult);
}
}
results = results.sort((a, b) => { return a.mean - b.mean });
console.log(`Rand\t${'Name'.padEnd(20)}Time (in milliseconds)`);
let id = 1;
for (let r of results) {
console.log(`${id++}\t${r.name.padEnd(20)}${r.mean.toFixed(3)}`);
}
})
.run({ async: true });

45
binding/nodejs/tests/constructorTest.spec.js → binding/nodejs/tests/unitTests/constructorTest.spec.js
View File

@ -1,7 +1,7 @@
// This test is used for tesing of a static function `create` of IP2Region
const IP2Region = require('../ip2region');
const testIps = require('./utils/testData');
const asyncFor = require('./utils/asyncFor');
const IP2Region = require('../../ip2region');
const testIps = require('../utils/testData');
const asyncFor = require('../utils/asyncFor');
describe('Constructor Test', () => {
let instance;
@ -26,6 +26,11 @@ describe('Constructor Test', () => {
}
});
test('memorySearchSync query', () => {
for (const ip of testIps) {
expect(instance.memorySearchSync(ip)).toMatchSnapshot();
}
});
//#region callBack
test('binarySearch query', (done) => {
@ -52,10 +57,22 @@ describe('Constructor Test', () => {
() => { done() });
});
test('memorySearch query', (done) => {
asyncFor(testIps,
(value, continueCallBack) => {
instance.memorySearch(value, (err, result) => {
expect(err).toBe(null);
expect(result).toMatchSnapshot();
continueCallBack();
});
},
() => { done() });
});
//#endregion
//#region Async Promisify test
const node_ver = require('./utils/fetchMainVersion');
const node_ver = require('../utils/fetchMainVersion');
// If we have Nodejs >= 8, we now support `async` and `await`
if (node_ver >= 8) {
@ -90,6 +107,19 @@ describe('Constructor Test', () => {
};
const asyncMemorySearch = async (ip) => {
return new Promise((succ, fail) => {
instance.memorySearch(ip, (err, result) => {
if (err) {
fail(err);
}
else {
succ(result);
}
});
});
}
test('async binarySearch query', async () => {
for (let i = 0; i < testIps.length; ++i) {
const result = await asyncBinarySearch(testIps[i]);
@ -103,6 +133,13 @@ describe('Constructor Test', () => {
expect(result).toMatchSnapshot();
}
});
test('async memorySearch query', async () => {
for (let i = 0; i < testIps.length; ++i) {
const result = await asyncMemorySearch(testIps[i]);
expect(result).toMatchSnapshot();
}
});
}
//#endregion
});

46
binding/nodejs/tests/createTest.spec.js → binding/nodejs/tests/unitTests/createTest.spec.js
View File

@ -1,7 +1,7 @@
// This test is used for tesing of a static function `create` of IP2Region
const IP2Region = require('../ip2region');
const testIps = require('./utils/testData');
const asyncFor = require('./utils/asyncFor');
const IP2Region = require('../../ip2region');
const testIps = require('../utils/testData');
const asyncFor = require('../utils/asyncFor');
describe('Create Test', () => {
let instance;
@ -26,6 +26,11 @@ describe('Create Test', () => {
}
});
test('memorySearchSync query', () => {
for (const ip of testIps) {
expect(instance.memorySearchSync(ip)).toMatchSnapshot();
}
});
//#region callBack
test('binarySearch query', (done) => {
@ -52,10 +57,23 @@ describe('Create Test', () => {
() => { done() });
});
test('memorySearch query', (done) => {
asyncFor(testIps,
(value, continueCallBack) => {
instance.memorySearch(value, (err, result) => {
expect(err).toBe(null);
expect(result).toMatchSnapshot();
continueCallBack();
});
},
() => { done() });
});
//#endregion
//#region Async Promisify test
const node_ver = require('./utils/fetchMainVersion');
const node_ver = require('../utils/fetchMainVersion');
// If we have Nodejs >= 8, we now support `async` and `await`
if (node_ver >= 8) {
@ -90,6 +108,19 @@ describe('Create Test', () => {
};
const asyncMemorySearch = async (ip) => {
return new Promise((succ, fail) => {
instance.memorySearch(ip, (err, result) => {
if (err) {
fail(err);
}
else {
succ(result);
}
});
});
}
test('async binarySearch query', async () => {
for (let i = 0; i < testIps.length; ++i) {
const result = await asyncBinarySearch(testIps[i]);
@ -103,6 +134,13 @@ describe('Create Test', () => {
expect(result).toMatchSnapshot();
}
});
test('async memorySearch query', async () => {
for (let i = 0; i < testIps.length; ++i) {
const result = await asyncMemorySearch(testIps[i]);
expect(result).toMatchSnapshot();
}
});
}
//#endregion
});

2
binding/nodejs/tests/exceptionTest.spec.js → binding/nodejs/tests/unitTests/exceptionTest.spec.js
View File

@ -1,6 +1,6 @@
// This test is used for tesing of exceptions
const IP2Region = require('../ip2region');
const IP2Region = require('../../ip2region');
describe('Constructor Test', () => {
let instance;