archive-gh-me/ip2region
1
0
Fork 0
mirror of https://github.com/lionsoul2014/ip2region.git synced 2025-12-08 19:25:22 +00:00
Code Issues Projects Releases Wiki Activity

feat:support for concurrent file-based queries

Browse Source
This commit is contained in:
Alan 2023-07-26 21:25:34 +08:00
parent c8446edaab
commit 7f5ec337d0
13 changed files with 259 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
binding/csharp/IP2Region.Net.BenchMark/Program.cs
View File

@ -1,13 +1,16 @@
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;
using IP2Region.Net.Abstractions;
using IP2Region.Net.XDB;
BenchmarkRunner.Run(typeof(Program).Assembly);
public class CachePolicyCompare
{
private readonly Searcher _contentSearcher = new Searcher(CachePolicy.Content);
private readonly Searcher _vectorSearcher = new Searcher(CachePolicy.VectorIndex);
private static readonly string XdbPath = Path.Combine(AppContext.BaseDirectory, "IP2Region", "ip2region.xdb");
private readonly ISearcher _contentSearcher = new Searcher(CachePolicy.Content, XdbPath);
private readonly ISearcher _vectorSearcher = new Searcher(CachePolicy.VectorIndex,XdbPath);
private readonly ISearcher _fileSearcher = new Searcher(CachePolicy.File,XdbPath);
private readonly string _testIpAddress = "114.114.114.114";
@ -18,4 +21,9 @@ public class CachePolicyCompare
[Benchmark]
[BenchmarkCategory(nameof(CachePolicy.VectorIndex))]
public void CachePolicy_VectorIndex() => _vectorSearcher.Search(_testIpAddress);
[Benchmark]
[BenchmarkCategory(nameof(CachePolicy.File))]
public void CachePolicy_File() => _fileSearcher.Search(_testIpAddress);
}

20
binding/csharp/IP2Region.Net/Abstractions/ISearcher.cs Normal file
View File

@ -0,0 +1,20 @@
// Copyright 2023 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// @Author Alan <lzh.shap@gmail.com>
// @Date 2023/07/25
using System.Net;
namespace IP2Region.Net.Abstractions;
public interface ISearcher
{
string? Search(string ipStr);
string? Search(IPAddress ipAddress);
string? Search(uint ipAddress);
int IoCount { get; }
}

16
binding/csharp/IP2Region.Net/CHANGELOG.md Normal file
View File

@ -0,0 +1,16 @@
# Changelog
All notable changes to this project will be documented in this file.
## [Unreleased]
## [2.0.0] - 2023-07-26
### Removed
- Remove nuget include xdb file
- Searcher cache policy default parameters
- Searcher xdb file path default parameters
### Added
- Dependent file query policies CachePolicy.VectorIndex, CachePolicy.File support thread-safe concurrent queries
- Dramatically optimizes overall performance

19
binding/csharp/IP2Region.Net/IP2Region.Net.csproj
View File

@ -2,32 +2,23 @@
<PropertyGroup>
<id>IP2Region.Net</id>
<version>1.0.2</version>
<version>2.0.0</version>
<title>IP2Region.Net</title>
<authors>Alan Lee</authors>
<PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
<PackageReadmeFile>README.md</PackageReadmeFile>
<PackageProjectUrl>https://github.com/lionsoul2014/ip2region/tree/master/binding/csharp</PackageProjectUrl>
<RepositoryUrl>https://github.com/lionsoul2014/ip2region/tree/master/binding/csharp</RepositoryUrl>
<Description>ip2region 是一个离线IP地址定位库，极速响应，微秒级查询</Description>
<tags>IP2Region</tags>
<PackageReleaseNotes>Please refer to CHANGELOG.md for details</PackageReleaseNotes>
<Description>.NET client library for ip2region</Description>
<PackageTags>IP2Region GeoIP IPSearch</PackageTags>
<RepositoryType>git</RepositoryType>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TargetFrameworks>net6.0;netstandard2.1</TargetFrameworks>
<LangVersion>10.0</LangVersion>
</PropertyGroup>
<ItemGroup>
<Folder Include="Data" />
</ItemGroup>
<ItemGroup>
<None Update="Data\ip2region.xdb">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="..\README.md" Pack="true" PackagePath="\" />
</ItemGroup>
</Project>

64
binding/csharp/IP2Region.Net/Internal/Abstractions/AbstractCacheStrategy.cs Normal file
View File

@ -0,0 +1,64 @@
// Copyright 2023 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// @Author Alan <lzh.shap@gmail.com>
// @Date 2023/07/25
using System.Buffers;
namespace IP2Region.Net.Internal.Abstractions;
internal abstract class AbstractCacheStrategy
{
protected const int HeaderInfoLength = 256;
protected const int VectorIndexRows = 256;
protected const int VectorIndexCols = 256;
protected const int VectorIndexSize = 8;
protected readonly FileStream XdbFileStream;
private const int BufferSize = 4096;
internal int IoCount { get; private set; }
protected AbstractCacheStrategy(string xdbPath)
{
XdbFileStream = new FileStream(xdbPath, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize,
useAsync: true);
}
protected int GetVectorIndexStartPos(uint ip)
{
var il0 = ip >> 24 & 0xFF;
var il1 = ip >> 16 & 0xFF;
var idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
return (int)idx;
}
internal abstract ReadOnlyMemory<byte> GetVectorIndex(uint ip);
internal virtual ReadOnlyMemory<byte> GetData(int offset, int length)
{
byte[] buffer = ArrayPool<byte>.Shared.Rent(length);
int totalBytesRead = 0;
try
{
XdbFileStream.Seek(offset, SeekOrigin.Begin);
int bytesRead;
do
{
int bytesToRead = Math.Min(BufferSize, length - totalBytesRead);
bytesRead = XdbFileStream.Read(buffer, totalBytesRead, bytesToRead);
totalBytesRead += bytesRead;
IoCount++;
} while (bytesRead > 0 && totalBytesRead < length);
}
finally
{
ArrayPool<byte>.Shared.Return(buffer);
}
return new ReadOnlyMemory<byte>(buffer, 0, totalBytesRead);
}
}

31
binding/csharp/IP2Region.Net/Internal/CacheStrategyFactory.cs Normal file
View File

@ -0,0 +1,31 @@
// Copyright 2023 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// @Author Alan <lzh.shap@gmail.com>
// @Date 2023/07/25
using IP2Region.Net.Internal.Abstractions;
using IP2Region.Net.XDB;
namespace IP2Region.Net.Internal;
internal class CacheStrategyFactory
{
private readonly string _xdbPath;
public CacheStrategyFactory(string xdbPath)
{
_xdbPath = xdbPath;
}
public AbstractCacheStrategy CreateCacheStrategy(CachePolicy cachePolicy)
{
return cachePolicy switch
{
CachePolicy.Content => new ContentCacheStrategy(_xdbPath),
CachePolicy.VectorIndex => new VectorIndexCacheStrategy(_xdbPath),
CachePolicy.File => new FileCacheStrategy(_xdbPath),
_ => throw new ArgumentException(nameof(cachePolicy))
};
}
}

32
binding/csharp/IP2Region.Net/Internal/ContentCacheStrategy.cs Normal file
View File

@ -0,0 +1,32 @@
// Copyright 2023 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// @Author Alan <lzh.shap@gmail.com>
// @Date 2023/07/25
using IP2Region.Net.Internal.Abstractions;
namespace IP2Region.Net.Internal;
internal class ContentCacheStrategy : AbstractCacheStrategy
{
private readonly ReadOnlyMemory<byte> _cacheData;
public ContentCacheStrategy(string xdbPath) : base(xdbPath)
{
_cacheData = base.GetData(0, (int)XdbFileStream.Length);
XdbFileStream.Close();
XdbFileStream.Dispose();
}
internal override ReadOnlyMemory<byte> GetVectorIndex(uint ip)
{
int idx = GetVectorIndexStartPos(ip);
return _cacheData.Slice(HeaderInfoLength + idx, VectorIndexSize);
}
internal override ReadOnlyMemory<byte> GetData(int offset, int length)
{
return _cacheData.Slice(offset, length);
}
}

22
binding/csharp/IP2Region.Net/Internal/FileCacheStrategy.cs Normal file
View File

@ -0,0 +1,22 @@
// Copyright 2023 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// @Author Alan <lzh.shap@gmail.com>
// @Date 2023/07/25
using IP2Region.Net.Internal.Abstractions;
namespace IP2Region.Net.Internal;
internal class FileCacheStrategy : AbstractCacheStrategy
{
public FileCacheStrategy(string xdbPath) : base(xdbPath)
{
}
internal override ReadOnlyMemory<byte> GetVectorIndex(uint ip)
{
var idx = GetVectorIndexStartPos(ip);
return GetData(HeaderInfoLength + idx, VectorIndexSize);
}
}

26
binding/csharp/IP2Region.Net/Internal/VectorIndexCacheStrategy.cs Normal file
View File

@ -0,0 +1,26 @@
// Copyright 2023 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// @Author Alan <lzh.shap@gmail.com>
// @Date 2023/07/25
using IP2Region.Net.Internal.Abstractions;
namespace IP2Region.Net.Internal;
internal class VectorIndexCacheStrategy : AbstractCacheStrategy
{
private readonly ReadOnlyMemory<byte> _vectorIndex;
public VectorIndexCacheStrategy(string xdbPath) : base(xdbPath)
{
var vectorLength = VectorIndexRows * VectorIndexCols * VectorIndexSize;
_vectorIndex = base.GetData(HeaderInfoLength, vectorLength);
}
internal override ReadOnlyMemory<byte> GetVectorIndex(uint ip)
{
var idx = GetVectorIndexStartPos(ip);
return _vectorIndex.Slice(idx, VectorIndexSize);
}
}

6
binding/csharp/IP2Region.Net/XDB/CachePolicy.cs
View File

@ -3,15 +3,15 @@ namespace IP2Region.Net.XDB;
public enum CachePolicy
{
/// <summary>
/// no cache , not thread safe!
/// no cache
/// </summary>
File,
/// <summary>
/// cache vector index , reduce the number of IO operations , not thread safe!
/// cache vector index , reduce the number of IO operations
/// </summary>
VectorIndex,
/// <summary>
/// default cache policy , cache whole xdb file , thread safe
/// default cache policy , cache whole xdb file
/// </summary>
Content
}

14
binding/csharp/IP2Region.Net/XDB/ISearcher.cs
View File

@ -1,14 +0,0 @@
using System.Net;
namespace IP2Region.Net.XDB;
public interface ISearcher
{
string? Search(string ipStr);
string? Search(IPAddress ipAddress);
string? Search(uint ipAddress);
int IoCount { get; }
}

134
binding/csharp/IP2Region.Net/XDB/Searcher.cs
View File

@ -1,60 +1,29 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Copyright 2023 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// @Author Alan Lee <lzh.shap@gmail.com>
// @Date 2022/09/06
// @Date 2023/07/23
using System.Net;
using System.Runtime.InteropServices;
using System.Text;
using IP2Region.Net.Abstractions;
using IP2Region.Net.Internal;
using IP2Region.Net.Internal.Abstractions;
namespace IP2Region.Net.XDB;
public class Searcher : ISearcher
{
const int HeaderInfoLength = 256;
const int VectorIndexRows = 256;
const int VectorIndexCols = 256;
const int VectorIndexSize = 8;
const int SegmentIndexSize = 14;
private readonly byte[]? _vectorIndex;
private readonly byte[]? _contentBuff;
private readonly FileStream _contentStream;
private readonly CachePolicy _cachePolicy;
public int IoCount { get; private set; }
private readonly AbstractCacheStrategy _cacheStrategy;
public int IoCount => _cacheStrategy.IoCount;
public Searcher(CachePolicy cachePolicy = CachePolicy.Content, string? dbPath = null)
public Searcher(CachePolicy cachePolicy, string dbPath)
{
if (string.IsNullOrEmpty(dbPath))
{
dbPath = Path.Combine(AppContext.BaseDirectory, "Data", "ip2region.xdb");
}
_contentStream = File.OpenRead(dbPath);
_cachePolicy = cachePolicy;
switch (_cachePolicy)
{
case CachePolicy.Content:
using (var stream = new MemoryStream())
{
_contentStream.CopyTo(stream);
_contentBuff = stream.ToArray();
}
break;
case CachePolicy.VectorIndex:
var vectorLength = VectorIndexRows * VectorIndexCols * VectorIndexSize;
_vectorIndex = new byte[vectorLength];
Read(HeaderInfoLength, _vectorIndex);
break;
}
}
~Searcher()
{
_contentStream.Close();
_contentStream.Dispose();
var factory = new CacheStrategyFactory(dbPath);
_cacheStrategy = factory.CreateCacheStrategy(cachePolicy);
}
public string? Search(string ipStr)
@ -71,62 +40,37 @@ public class Searcher : ISearcher
public string? Search(uint ip)
{
var il0 = ip >> 24 & 0xFF;
var il1 = ip >> 16 & 0xFF;
var idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
uint sPtr = 0, ePtr = 0;
switch (_cachePolicy)
{
case CachePolicy.VectorIndex:
sPtr = BitConverter.ToUInt32(_vectorIndex.AsSpan()[(int)idx..]);
ePtr = BitConverter.ToUInt32(_vectorIndex.AsSpan()[((int)idx + 4)..]);
break;
case CachePolicy.Content:
sPtr = BitConverter.ToUInt32(_contentBuff.AsSpan()[(HeaderInfoLength + (int)idx)..]);
ePtr = BitConverter.ToUInt32(_contentBuff.AsSpan()[(HeaderInfoLength + (int)idx + 4)..]);
break;
case CachePolicy.File:
var buff = new byte[VectorIndexSize];
Read((int)(idx + HeaderInfoLength), buff);
sPtr = BitConverter.ToUInt32(buff);
ePtr = BitConverter.ToUInt32(buff.AsSpan()[4..]);
break;
}
var index = _cacheStrategy.GetVectorIndex(ip);
uint sPtr = MemoryMarshal.Read<uint>(index.Span);
uint ePtr = MemoryMarshal.Read<uint>(index.Span[4..]);
var dataLen = 0;
uint dataPtr = 0;
var l = 0;
var h = (int)((ePtr - sPtr) / SegmentIndexSize);
var buffer = new byte[SegmentIndexSize];
uint l = 0;
uint h = (ePtr -sPtr) / SegmentIndexSize;
while (l <= h)
{
var mid = Util.GetMidIp(l, h);
var pos = sPtr + mid * SegmentIndexSize;
Read((int)pos, buffer);
var sip = BitConverter.ToUInt32(buffer);
var buffer = _cacheStrategy.GetData((int)pos, SegmentIndexSize);
uint sip = MemoryMarshal.Read<uint>(buffer.Span);
uint eip = MemoryMarshal.Read<uint>(buffer.Span[4..]);
if (ip < sip)
{
h = mid - 1;
}
else if (ip > eip)
{
l = mid + 1;
}
else
{
var eip = BitConverter.ToUInt32(buffer.AsSpan()[4..]);
if (ip > eip)
{
l = mid + 1;
}
else
{
dataLen = BitConverter.ToUInt16(buffer.AsSpan()[8..]);
dataPtr = BitConverter.ToUInt32(buffer.AsSpan()[10..]);
break;
}
dataLen = MemoryMarshal.Read<ushort>(buffer.Span[8..]);
dataPtr = MemoryMarshal.Read<uint>(buffer.Span[10..]);
break;
}
}
@ -135,29 +79,7 @@ public class Searcher : ISearcher
return default;
}
var regionBuff = new byte[dataLen];
Read((int)dataPtr, regionBuff);
return Encoding.UTF8.GetString(regionBuff);
}
private void Read(int offset, byte[] buff)
{
switch (_cachePolicy)
{
case CachePolicy.Content:
_contentBuff.AsSpan()[offset..(offset + buff.Length)].CopyTo(buff);
break;
default:
_contentStream.Seek(offset, SeekOrigin.Begin);
IoCount++;
var rLen = _contentStream.Read(buff);
if (rLen != buff.Length)
{
throw new IOException($"incomplete read: readed bytes should be {buff.Length}");
}
break;
}
var regionBuff = _cacheStrategy.GetData((int)dataPtr,dataLen);
return Encoding.UTF8.GetString(regionBuff.Span);
}
}

6
binding/csharp/IP2Region.Net/XDB/Util.cs
View File

@ -1,4 +1,5 @@
using System.Net;
using System.Runtime.InteropServices;
namespace IP2Region.Net.XDB;
@ -14,12 +15,9 @@ public static class Util
{
byte[] bytes = ipAddress.GetAddressBytes();
Array.Reverse(bytes);
return BitConverter.ToUInt32(bytes, 0);
return MemoryMarshal.Read<uint>(bytes);
}
public static uint GetMidIp(uint x, uint y)
=> (x & y) + ((x ^ y) >> 1);
public static int GetMidIp(int x, int y)
=> (x & y) + ((x ^ y) >> 1);
}