Merge pull request #282 from leihua996/master

add xdb erlang implementation

.gitignore

@@ -81,3 +81,7 @@ target
 /maker/golang/dbmaker
 /maker/golang/xdb_maker
 /maker/golang/golang
+
+#erlang
+/binding/erlang/_build
+/binding/erlang/doc

binding/erlang/README.md

@@ -0,0 +1,129 @@
+# ip2region xdb erlang 查询客户端
+
+### 简介
+该bingding以erlang语言实现xdb查询客户端，基于Erlang OTP Application，查询逻辑由ip2region_worker工作进程实现，支持配多个工作进程来进行负载均衡。
+
+### 应用配置
+该应用可配置的参数在ip2region.app.src中,如下：
+``` erlang
+  {env,[
+    {poolargs, [
+        {size, 1},  %% 工作进程默认数量
+        {max_overflow, 5}  %% 工作进程最大数量
+    ]}
+  ]}
+```
+
+### 编译
+
+```
+$ rebar3 compile
+```
+### 运行
+将xdb文件放到priv目录下，然后启动erlang节点：
+```
+$ rebar3 shell
+
+```
+在erlang shell中调用xdb:search/1接口查询Ip地址信息, 该接口支持以list格式字符串、bianry格式字符串、tuple和整数表示的IP地址，如下：
+```
+1> xdb:search("1.0.8.0").
+[20013,22269,124,48,124,24191,19996,30465,124,24191,24030,
+ 24066,124,30005,20449]
+2>
+3> io:format("~ts~n", [xdb:search("1.0.8.0")]).
+中国|0|广东省|广州市|电信
+io:format("~ts~n", [xdb:search(<<"1.0.8.0">>)]).
+中国|0|广东省|广州市|电信
+4> io:format("~ts~n", [xdb:search({1,0,8,0})]).
+中国|0|广东省|广州市|电信
+6> io:format("~ts~n", [xdb:search(16779264)]).
+中国|0|广东省|广州市|电信
+```
+
+### 使用方法
+* 在rebar.config中引入依赖
+```
+{deps, [
+  ip2region
+]}.
+
+```
+* 启动ip2region Application
+```
+......
+
+application:ensure_started(ip2region),
+
+......
+```
+
+* 调用xdb:search/1接口查询IP信息
+```
+......
+
+ip2region:search("1.0.8.0"),
+
+......
+```
+
+### 单元测试
+
+```
+$ rebar3 eunit
+===> Verifying dependencies...
+===> Analyzing applications...
+===> Compiling ip2region
+===> Performing EUnit tests...
+=INFO REPORT==== 17-Jan-2023::11:52:59.920155 ===
+XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/test/lib/ip2region/priv/ip2region.xdb
+
+....
+Finished in 0.074 seconds
+4 tests, 0 failures
+
+```
+
+### 基准测试
+```
+$ cd benchmarks/
+$ sh xdb-benchmark.sh
+===> Verifying dependencies...
+===> Analyzing applications...
+===> Compiling ip2region
+Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit]
+
+Eshell V12.3.2.2  (abort with ^G)
+1> =INFO REPORT==== 17-Jan-2023::11:37:35.631095 ===
+XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/default/lib/ip2region/priv/ip2region.xdb
+
+===> Booted ip2region
+===> Evaluating: "xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()."
+CPU info:
+model name      : AMD EPYC 7K62 48-Core Processor
+cache size      : 512 KB
+cpu MHz         : 2595.124
+bogomips        : 5190.24
+cores/threads   : 2
+
+Erlang info:
+system_version:Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit]
+load test data use 4.835593s
+
+start run benchmark tests
+
+search from file:
+ip count:683844,
+total time: 28.201699s,
+search 24248.326315375536 times per second,
+use 41.23995969841075 micro second per search
+
+search from cache:
+ip count:683844,
+total time: 0.671801s,
+search 1017926.4395259906 times per second,
+use 0.9823892583688677 micro second per search
+
+benchmark test finish
+
+```

binding/erlang/benchmarks/xdb-benchmark.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd ..
+
+rebar3 shell --eval="xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()."

binding/erlang/include/ip2region.hrl

@@ -0,0 +1,26 @@
+-ifndef(IP2REGION_HRL).
+-define(IP2REGION_HRL, true).
+
+-define(NONE, none).
+-define(APP_NAME, ip2region).
+
+-define(XDB_VECTOR_INDEX, ets_xdb_vector_index).
+-define(XDB_SEGMENT_INDEX, ets_xdb_segement_index).
+-define(IP2REGION_CACHE, ets_ip2region_cache).
+
+
+-define(XDB_HEADER_SIZE, 256).
+-define(XDB_VECTOR_COLS, 256).
+-define(XDB_VECTOR_INDEX_SIZE, 8). 
+-define(XDB_VECTOR_INDEX_COUNT, (16#10000)). %% 256*256
+
+-define(XDB_SEGMENT_INDEX_SIZE, 14).
+
+-define(IP2REGION_POOL, ip2region_pool).
+
+-ifndef(IF).
+-define(IF(C, T, F), case (C) of true -> (T); false -> (F) end).
+-define(IF(C, T), ?IF(C, T, skip)).
+-endif.
+
+-endif.

binding/erlang/rebar.config

@@ -0,0 +1,24 @@
+{erl_opts, [
+    debug_info, 
+    export_all, 
+    nowarn_export_all
+]}.
+
+{plugins, [rebar3_hex, rebar3_ex_doc]}.
+
+{deps, [
+    poolboy
+]}.
+
+{shell, [
+  % {config, "config/sys.config"},
+    {apps, [ip2region]}
+]}.
+
+{ex_doc, [
+    {extras, ["README.md"]},
+    {main, "README.md"},
+    {source_url, "https://github.com/leihua996/ip2region/tree/master/binding/erlang"}
+]}.
+
+{hex, [{doc, ex_doc}]}.

binding/erlang/rebar.lock

@@ -0,0 +1,8 @@
+{"1.2.0",
+[{<<"poolboy">>,{pkg,<<"poolboy">>,<<"1.5.2">>},0}]}.
+[
+{pkg_hash,[
+ {<<"poolboy">>, <<"392B007A1693A64540CEAD79830443ABF5762F5D30CF50BC95CB2C1AAAFA006B">>}]},
+{pkg_hash_ext,[
+ {<<"poolboy">>, <<"DAD79704CE5440F3D5A3681C8590B9DC25D1A561E8F5A9C995281012860901E3">>}]}
+].

binding/erlang/src/ip2region.app.src

@@ -0,0 +1,20 @@
+{application, ip2region,
+ [{description, "ip2region xdb client application"},
+  {vsn, "0.1.0"},
+  {registered, []},
+  {mod, {ip2region_app, []}},
+  {applications,
+   [kernel,
+    stdlib
+   ]},
+  {env,[
+    {poolargs, [
+        {size, 1},
+        {max_overflow, 5}
+    ]}
+  ]},
+  {modules, []},
+
+  {licenses, ["Apache-2.0"]},
+  {links, [{"Github", "https://github.com/leihua996/ip2region/tree/master/binding/erlang"}]}
+ ]}.

binding/erlang/src/ip2region_app.erl

@@ -0,0 +1,22 @@
+%%%-------------------------------------------------------------------
+%% Copyright 2022 The Ip2Region Authors. All rights reserved.
+%% Use of this source code is governed by a Apache2.0-style
+%% license that can be found in the LICENSE file.
+%% 
+%% @doc 
+%% @end
+%%%-------------------------------------------------------------------
+
+-module(ip2region_app).
+
+-behaviour(application).
+
+-export([start/2, stop/1]).
+
+start(_StartType, _StartArgs) ->
+    ip2region_sup:start_link().
+
+stop(_State) ->
+    ok.
+
+%% internal functions

binding/erlang/src/ip2region_sup.erl

@@ -0,0 +1,55 @@
+%%%-------------------------------------------------------------------
+%% Copyright 2022 The Ip2Region Authors. All rights reserved.
+%% Use of this source code is governed by a Apache2.0-style
+%% license that can be found in the LICENSE file.
+%% 
+%% @doc ip2region top level supervisor.
+%% @end
+%%%-------------------------------------------------------------------
+-module(ip2region_sup).
+-behaviour(supervisor).
+-include("ip2region.hrl").
+
+-export([start_link/0]).
+
+-export([init/1, create_table/0]).
+
+-define(SERVER, ?MODULE).
+
+start_link() ->
+    {ok, SupPid} = supervisor:start_link({local, ?SERVER}, ?MODULE, []),
+    {ok, _PoolPid} = start_ip2region_pool(SupPid),
+    {ok, SupPid}.
+
+%% sup_flags() = #{strategy => strategy(),         % optional
+%%                 intensity => non_neg_integer(), % optional
+%%                 period => pos_integer()}        % optional
+%% child_spec() = #{id => child_id(),       % mandatory
+%%                  start => mfargs(),      % mandatory
+%%                  restart => restart(),   % optional
+%%                  shutdown => shutdown(), % optional
+%%                  type => worker(),       % optional
+%%                  modules => modules()}   % optional
+init([]) ->
+    create_table(),
+    SupFlags = #{strategy => one_for_one,
+                 intensity => 10,
+                 period => 5},
+    ChildSpecs = [],
+    {ok, {SupFlags, ChildSpecs}}.
+
+%% internal functions
+%% 
+create_table() ->
+    Opts = [named_table, set, public, {read_concurrency, true}, {keypos, 1}],
+    ets:new(?XDB_VECTOR_INDEX, Opts),
+    ets:new(?XDB_SEGMENT_INDEX, Opts),
+    ets:new(?IP2REGION_CACHE, Opts).
+
+start_ip2region_pool(Sup) ->
+    {ok, PoolArgsCfg} = application:get_env(poolargs),
+    PoolName = ?IP2REGION_POOL,
+    PoolArgs = [{strategy, fifo}, {name, {local, PoolName}}, {worker_module, ip2region_worker} | PoolArgsCfg],
+    WorkerArgs = [],
+    ChildSpecs = poolboy:child_spec(PoolName, PoolArgs, WorkerArgs),
+    supervisor:start_child(Sup, ChildSpecs).

binding/erlang/src/ip2region_util.erl

@@ -0,0 +1,26 @@
+%%%-------------------------------------------------------------------
+%% Copyright 2022 The Ip2Region Authors. All rights reserved.
+%% Use of this source code is governed by a Apache2.0-style
+%% license that can be found in the LICENSE file.
+%% 
+%% @doc 
+%% ip2region utils
+%% @end
+%%%-------------------------------------------------------------------
+-module(ip2region_util).
+-export([ipv4_to_n/1]).
+
+
+ipv4_to_n(IntIp) when is_integer(IntIp) -> IntIp;
+ipv4_to_n({A, B, C, D}) ->
+	<<N:32>> = <<A, B, C, D>>,
+	N;
+ipv4_to_n(Ip) when is_binary(Ip) ->
+	ipv4_to_n(binary_to_list(Ip));
+ipv4_to_n(Ip) when is_list(Ip) ->
+    case inet_parse:address(Ip) of
+        {ok, Addr} ->
+            ipv4_to_n(Addr);
+        _ ->
+            {error, bad_ip_format}
+    end.

binding/erlang/src/ip2region_worker.erl

@@ -0,0 +1,183 @@
+%%%-------------------------------------------------------------------
+%% Copyright 2022 The Ip2Region Authors. All rights reserved.
+%% Use of this source code is governed by a Apache2.0-style
+%% license that can be found in the LICENSE file.
+%% 
+%% @doc 
+%% ip2region xdb client worker
+%% @end
+%%%-------------------------------------------------------------------
+-module(ip2region_worker).
+-behaviour(gen_server).
+-include("ip2region.hrl").
+
+%% API
+-export([start/1, stop/1, start_link/1]).
+-export([search/2]).
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
+-record(state, {xdb_fd}).
+
+%%==========================================
+%% API
+%% =========================================
+start(Args) ->
+    Opts = [{spawn_opt, [{min_heap_size, 6000}]}],
+    gen_server:start(?MODULE, Args, Opts).
+
+start_link(Args) ->
+    Opts = [{spawn_opt, [{min_heap_size, 6000}]}],
+    gen_server:start_link(?MODULE, Args, Opts).
+
+
+stop(Pid) ->
+    gen_server:call(Pid, stop).
+
+search(Pid, Ip) ->
+    gen_server:call(Pid, {search, Ip}).
+
+%%==========================================
+%% gen_server callbacks
+%% =========================================
+init(_Args) ->
+    process_flag(trap_exit, true),
+    AppName = 
+        case application:get_application() of
+            {ok, AName} -> AName;
+            _ -> ?APP_NAME
+        end,
+    PrivDir = code:priv_dir(AppName),
+    XdbFileName = filename:join([PrivDir, "ip2region.xdb"]),
+    error_logger:info_report(io_lib:format("XdbFile:~s~n", [XdbFileName])),
+    {ok, IoDevice} = file:open(XdbFileName, [read, binary]),
+    load_vector_index(IoDevice),
+    {ok, #state{xdb_fd = IoDevice}}.
+
+handle_call(Request, From, State) ->
+    try
+        do_call(Request, From, State)
+    catch
+        Class:Error:Stacktrace ->
+            error_logger:error_report(io_lib:format("~p handle call error, Req:~p ~p, stacktrace:~p~n", 
+                [?MODULE, Request, {Class, Error}, Stacktrace])),
+            {reply, {error, {Class, Error}}, State}
+    end.
+
+handle_cast(Msg, State) ->
+    try
+        do_cast(Msg, State)
+    catch
+        Class:Error:Stacktrace ->
+            error_logger:error_report(io_lib:format("~p handle cast error, Msg:~p, ~p, stacktrace:~w~n", 
+                [?MODULE, Msg, {Class, Error}, Stacktrace])),
+            {noreply, State}
+    end.
+
+handle_info(Info, State) ->
+    try
+        do_info(Info, State)
+    catch
+        Class:Error:Stacktrace ->
+            error_logger:error_report(io_lib:format("~p handle info error, Info:~p, ~p, stacktrace:~p~n", 
+                [?MODULE, Info, {Class, Error}, Stacktrace])),
+            {noreply, State}
+    end.
+
+terminate(_Reason, State) ->
+    #state{xdb_fd = XdbFd} = State,
+    case is_pid(XdbFd) of
+        true ->
+            file:close(XdbFd);
+        _ ->
+            skip
+    end,
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+
+%%==========================================
+%% Internal function
+%% =========================================
+do_call({search, Ip}, _From, #state{xdb_fd = IoDevice} = State) ->
+    Reply = search_ip(IoDevice, Ip),
+    {reply, Reply, State};
+
+do_call(stop, _From, State) ->
+    {stop, normal, stopped, State};
+
+do_call(Request, From, State) ->
+    error_logger:error_report(io_lib:format("unknown request: ~p, from:~p", [Request, From])),
+    {noreply, State}.
+
+do_cast(Msg, State) ->
+    error_logger:error_report(io_lib:format("unknown msg: ~p", [Msg])),
+    {noreply, State}.
+
+do_info(Info, State) ->
+    error_logger:error_report(io:format("unknown info: ~p", [Info])),
+    {noreply, State}.
+    
+
+load_vector_index(IoDevice) ->
+    Key = ip2region_header_loaded,
+    case persistent_term:get(Key, false) of
+        true -> ok;
+        _ -> 
+            {ok, <<_Header:?XDB_HEADER_SIZE/binary, VectorIndexBin/binary>> } = 
+                file:read(IoDevice, ?XDB_HEADER_SIZE + ?XDB_VECTOR_INDEX_COUNT*8),
+            load_vector_index_aux(VectorIndexBin, 0),
+            persistent_term:put(Key, true)
+    end.
+
+load_vector_index_aux(<<>>, _Index) -> ok;
+load_vector_index_aux(<<SPtr:32/little, EPtr:32/little, VectorIndexBin/binary>>, Index) ->
+    Term = {Index, SPtr, EPtr},
+    ets:insert(?XDB_VECTOR_INDEX, Term),
+    load_vector_index_aux(VectorIndexBin, Index + 1).
+
+
+search_ip(IoDevice, Ip) ->
+    IntIp = ip2region_util:ipv4_to_n(Ip),
+    case ets:lookup(?IP2REGION_CACHE, IntIp) of
+        [{_IntIp, RegionInfo}] ->
+            RegionInfo;
+        _ ->
+            <<A:8, B:8, _Rest/binary>> = <<IntIp:32>>,
+            VectorIdx = A * ?XDB_VECTOR_COLS + B,
+            [{_, SPtr, EPtr}] = ets:lookup(?XDB_VECTOR_INDEX, VectorIdx),
+            RegionInfo = search_ip(IoDevice, IntIp, SPtr, EPtr, 0, (EPtr - SPtr) div ?XDB_SEGMENT_INDEX_SIZE),
+            ets:insert_new(?IP2REGION_CACHE, {IntIp, RegionInfo}),
+            RegionInfo
+    end.
+
+search_ip(IoDevice, IntIp, SPtr, EPtr, Low, High) when Low =< High ->
+    Middle = (Low + High) bsr 1,
+    SPtr2 = SPtr + Middle * ?XDB_SEGMENT_INDEX_SIZE,
+    {SIp, EIp, DataLen, DataPtr} = read_segement_index(IoDevice, SPtr2),
+    if 
+        IntIp < SIp ->
+            search_ip(IoDevice, IntIp, SPtr, EPtr, Low, Middle - 1);
+        IntIp > EIp ->
+            search_ip(IoDevice, IntIp, SPtr, EPtr, Middle + 1, High);
+        true ->
+            {ok, DataBin} = read_file(IoDevice, DataPtr, DataLen),
+            unicode:characters_to_nfc_list(DataBin)
+    end;
+search_ip(_IoDevice, _IntIp, _SPtr, _EPtr, _Low, _High)  ->
+    {error, unknown}.
+
+read_file(IoDevice, Position, DataLength) ->
+    file:position(IoDevice, {bof, Position}),
+    file:read(IoDevice, DataLength).
+
+read_segement_index(IoDevice, SPtr) ->
+    case ets:lookup(?XDB_SEGMENT_INDEX, SPtr) of
+        [{_SPtr, SIp, EIp, DataLen, DataPtr}] ->
+            {SIp, EIp, DataLen, DataPtr};
+        _ ->
+            {ok, <<SIp:32/little, EIp:32/little, DataLen:16/little, DataPtr:32/little>>} = 
+	            read_file(IoDevice, SPtr, ?XDB_SEGMENT_INDEX_SIZE),
+            ets:insert_new(?XDB_SEGMENT_INDEX, {SPtr, SIp, EIp, DataLen, DataPtr}),
+            {SIp, EIp, DataLen, DataPtr}
+    end.

binding/erlang/src/xdb.erl

@@ -0,0 +1,32 @@
+
+%%%-------------------------------------------------------------------
+%% Copyright 2022 The Ip2Region Authors. All rights reserved.
+%% Use of this source code is governed by a Apache2.0-style
+%% license that can be found in the LICENSE file.
+%% 
+%% @doc 
+%% ip2region xdb client search api
+%% @end
+%%%-------------------------------------------------------------------
+-module(xdb).
+-include("ip2region.hrl").
+
+-export([search/1]).
+
+-spec search(Ip :: tuple() | list() | binary()) -> Result :: binary | {error, Reason::atom()}.
+search(Ip) when is_integer(Ip); is_list(Ip); is_tuple(Ip); is_binary(Ip) ->
+    case ip2region_util:ipv4_to_n(Ip) of
+    IntIp when is_integer(IntIp) ->
+        case ets:lookup(?IP2REGION_CACHE, IntIp) of
+        [{_IntIp, Region}] -> Region;
+        _ ->
+            Worker = poolboy:checkout(?IP2REGION_POOL, true, infinity),
+            try
+                ip2region_worker:search(Worker, IntIp)
+            after
+                poolboy:checkin(?IP2REGION_POOL, Worker)
+            end
+        end;
+    Ret ->
+        Ret
+    end.

binding/erlang/src/xdb_benchmark.erl

@@ -0,0 +1,75 @@
+%%%-------------------------------------------------------------------
+%% Copyright 2022 The Ip2Region Authors. All rights reserved.
+%% Use of this source code is governed by a Apache2.0-style
+%% license that can be found in the LICENSE file.
+%% 
+%% @doc 
+%% ip2region xdb client benchmark test
+%% @end
+%%%-------------------------------------------------------------------
+-module(xdb_benchmark).
+-export([main/1]).
+
+main(DataFile) ->
+	application:ensure_started(ip2region),
+    show_hw_sw_info(),
+    IpList = load_test_data(DataFile),
+    run(IpList).
+
+show_hw_sw_info() ->
+    io:format("CPU info:~n", []),
+    io:format("~s", [os:cmd("egrep '^model name' /proc/cpuinfo | head -1")]),
+    io:format("~s", [os:cmd("egrep '^cache' /proc/cpuinfo | head -1")]),
+    io:format("~s", [os:cmd("egrep '^cpu MHz' /proc/cpuinfo | head -1")]),
+    io:format("~s", [os:cmd("egrep '^bogomips' /proc/cpuinfo | head -1")]),
+    io:format("cores/threads   : ~s~n", [os:cmd("egrep -c '^processor' /proc/cpuinfo")]),
+    io:format("Erlang info:~n", []),
+    io:format("system_version:~s", [erlang:system_info(system_version)]),
+    ok.
+
+load_test_data(DataFile) ->
+    {ok, Fd} = file:open(DataFile, [read]),
+    T0 = os:timestamp(),
+    IpList = load_test_data(Fd, []),
+    T1 = os:timestamp(),
+    Sec = timer:now_diff(T1, T0) / 1000000,
+    io:format("load test data use ~ps~n", [Sec]),
+    IpList.
+
+load_test_data(Fd, IpList) ->
+    case file:read_line(Fd) of
+        {ok, Ip} -> 
+            case string:tokens(unicode:characters_to_list(Ip), "|") of
+            [SIp | _Tail] ->
+                load_test_data(Fd, [string:trim(SIp)| IpList]);
+            _ ->
+                load_test_data(Fd, IpList)
+            end;
+        _ ->
+			file:close(Fd),
+            IpList
+    end.
+
+run(IpList) ->
+    garbage_collect(),
+    io:format("~nstart run benchmark tests~n", []),
+    io:format("~nsearch from file:~n", []),
+    run_test(IpList),
+    io:format("~nsearch from cache:~n", []),
+    run_test(IpList),
+    io:format("~nbenchmark test finish~n", []).
+
+run_test(IpList) ->
+    T0 = os:timestamp(),
+    run_test_aux(IpList),
+    T1 = os:timestamp(),
+    Sec = timer:now_diff(T1, T0) / 1000000,
+    IpCount = length(IpList),
+    io:format("ip count:~p,~ntotal time: ~ps,~nsearch ~p times per second,~nuse ~p micro second per search~n", 
+        [IpCount, Sec, IpCount / Sec, Sec * 1000000/IpCount]).
+
+run_test_aux([]) -> ok;
+run_test_aux([Ip | Tail]) ->
+    xdb:search(Ip),
+    run_test_aux(Tail).
+

binding/erlang/test/xdb_test.erl

@@ -0,0 +1,18 @@
+-module(xdb_test).
+
+-include_lib("eunit/include/eunit.hrl").
+
+search_test_() ->
+    application:ensure_started(ip2region),
+    A = "中国|0|广东省|广州市|电信",
+    Region0 = xdb:search("1.0.8.0"),
+    Region1 = xdb:search(<<"1.0.8.0">>),
+    Region2 = xdb:search({1,0,8,0}),
+    Region3 = xdb:search("xxx.0.8.0"),
+    [
+        ?_assert(A =:= Region0),
+        ?_assert(A =:= Region1),
+        ?_assert(A =:= Region2),
+        ?_assert({error, bad_ip_format} =:= Region3)
+
+    ].