verify api and verify the xdb before search or bench test

binding/lua/ReadMe.md

@@ -21,6 +21,23 @@ local version = xdb.IPv6
 -- 备注：以下演示直接使用 dbPath 和 version 变量
 ```
 
+### 文件验证
+建议您主动去验证 xdb 文件的适用性，因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件，验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证，例如在服务启动的时候，或者手动调用命令验证确认版本匹配即可，不要在每次创建的 Searcher 的时候运行验证，这样会影响查询的响应速度，尤其是高并发的使用场景。
+```lua
+local xdb = require('xdb_searcher')
+
+local err = xdb.verify(dbPath);
+if err ~= nil then
+    -- 适用性验证失败！！！
+    -- 当前查询客户端实现不适用于 dbPath 指定的 xdb 文件的查询.
+    -- 应该停止启动服务，使用合适的 xdb 文件或者升级到适合 dbPath 的 Searcher 实现。
+    print(string.format("binding is not applicable for xdb file '%s': %s", dbPath, err))
+    return
+end
+
+-- 验证通过，当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作
+```
+
 ### 完全基于文件的查询
 ```lua
 local xdb = require("xdb_searcher")

binding/lua/bench_test.lua

@@ -62,19 +62,40 @@ if string.len(dbFile) < 2 or string.len(srcFile) < 2 then
     return
 end
 
--- load the header and define the ip version
-local header, err = xdb.load_header(dbFile)
+-- open the dbFile
+local handle, closer, err = xdb.open_file(dbFile, "rb")
 if err ~= nil then
+    print(string.format("failed to open %s: %s", dbFile, err))
+    return
+end
+
+-- verify the xdb
+err = xdb.verify(handle)
+if err ~= nil then
+    closer()
+    print(string.format("verify(%s): %s", dbFile, err))
+    return
+end
+
+-- load the header and define the ip version
+local header, err = xdb.load_header(handle)
+if err ~= nil then
+    closer()
     print(string.format("failed to load the header: %s", err))
     return
 end
 
 local version, err = xdb.version_from_header(header)
 if err ~= nil then
+    closer()
     print(string.format("failed to detect version from header: %s", err))
     return
 end
 
+-- file close
+closer("bench_test")
+
+
 -- create the searcher based on the cache-policy
 local searcher, v_index, content
 if cachePolicy == "file" then

binding/lua/search_test.lua

@@ -59,19 +59,40 @@ if string.len(dbFile) < 2 then
     return
 end
 
--- load the header and define the ip version
-local header, err = xdb.load_header(dbFile)
+-- open the dbFile
+local handle, closer, err = xdb.open_file(dbFile, "rb")
 if err ~= nil then
+    print(string.format("failed to open %s: %s", dbFile, err))
+    return
+end
+
+-- verify the xdb
+err = xdb.verify(handle)
+if err ~= nil then
+    closer("xdb_verify")
+    print(string.format("verify(%s): %s", dbFile, err))
+    return
+end
+
+-- load the header and define the ip version
+local header, err = xdb.load_header(handle)
+if err ~= nil then
+    closer()
     print(string.format("failed to load the header: %s", err))
     return
 end
 
 local version, err = xdb.version_from_header(header)
 if err ~= nil then
+    closer()
     print(string.format("failed to detect version from header: %s", err))
     return
 end
 
+-- file close
+closer("search_test")
+
+
 -- create the searcher based on the cache-policy
 local searcher, v_index, content
 if cachePolicy == "file" then

binding/lua/util_test.lua

@@ -97,7 +97,7 @@ function test_load_vector_index()
     if err ~= nil then
         print("failed to load vector index: ", err)
     else
-        print("xdb vector index buffer loaded")
+        print("xdb vector index buffer loaded, length=", #v_index)
     end
 end
 
@@ -106,10 +106,25 @@ function test_load_content()
     if err ~= nil then
         print("failed to load content: ", err)
     else
-        print("xdb content buffer loaded")
+        print("xdb content buffer loaded, length=", #c_buffer)
     end
 end
 
+function test_verify()
+    local xdb_files = {
+        "../../data/ip2region_v4.xdb",
+        "../../data/ip2region_v6.xdb"
+    }
+
+    for _, path in ipairs(xdb_files) do
+        local err = xdb.verify(path)
+        if err ~= nil then
+            print(string.format("verify(%s): %s", path, err))
+        else
+            print(string.format("verify(%s): Ok", path))
+        end
+    end
+end
 
 function test_ip_search()
     local test_list = {

binding/lua/xdb_searcher.lua

@@ -232,25 +232,45 @@ end
 
 -- static util functions
 
+function xdb.open_file(db_path, mode)
+    local t, handle = type(db_path), nil
+    local _closer = nil
+    if t == "userdata" then
+        handle = db_path   -- file handle
+        _closer = function(caller) end
+    elseif t == "string" then
+        handle = io.open(db_path, mode)
+        if handle == nil then
+            return nil, nil, string.format("failed to open xdb file `%s`", db_path)
+        end
+
+        _closer = function(caller) 
+            handle:close()
+        end
+    end
+
+    return handle, _closer, nil
+end
+
 function xdb.load_header(db_path)
-    local handle = io.open(db_path, "r")
-    if handle == nil then
-        return nil, string.format("failed to open xdb file `%s`", db_path)
+    local handle, closer, err = xdb.open_file(db_path, "rb")
+    if err ~= nil then
+        return nil, err
     end
 
     local r = handle:seek("set", 0)
     if r == nil then
-        handle:close()
+        closer("load_header")
         return nil, "failed to seek to 0"
     end
 
     local c = handle:read(header_info_length)
     if c == nil then
-        handle:close()
+        closer("load_header")
         return nil, string.format("failed to read %d bytes", header_info_length)
     end
 
-    handle:close()
+    closer("load_header")
     return {
         ["version"] = le_get_uint16(c, 1),
         ["index_policy"] = le_get_uint16(c, 3),
@@ -267,43 +287,83 @@ function xdb.load_header(db_path)
 end
 
 function xdb.load_vector_index(db_path)
-    local handle = io.open(db_path, "r")
-    if handle == nil then
-        return nil, string.format("failed to open xdb file `%s`", db_path)
+    local handle, closer, err = xdb.open_file(db_path, "rb")
+    if err ~= nil then
+        return nil, err
     end
 
     local r = handle:seek("set", header_info_length)
     if r == nil then
-        handle:close()
+        closer("load_vector_index")
         return nil, string.format("failed to seek to %d", header_info_length)
     end
 
     local c = handle:read(vector_index_length)
     if c == nil then
-        handle:close()
+        closer("load_vector_index")
         return nil, string.format("failed to read %d bytes", vector_index_length)
     end
 
-    handle:close()
+    closer("load_vector_index")
     return c, nil
 end
 
 function xdb.load_content(db_path)
-    local handle = io.open(db_path, "r")
-    if handle == nil then
-        return nil, string.format("failed to open xdb file `%s`", db_path)
-    end
-
+    local handle, closer, err = xdb.open_file(db_path, "rb")
     local c = handle:read("*a")
     if c == nil then
-        handle:close()
+        closer("load_content")
         return nil, string.format("failed to read xdb content")
     end
 
-    handle:close()
+    closer("load_content")
     return c, nil
 end
 
+-- Verify if the current Searcher could be used to search the specified xdb file.
+-- Why do we need this check ?
+-- The future features of the xdb impl may cause the current searcher not able to work properly.
+--
+-- @Note: You Just need to check this ONCE when the service starts
+-- Or use another process (eg, A command) to check once Just to confirm the suitability.
+function xdb.verify(db_path)
+    local handle, closer, err = xdb.open_file(db_path, "rb")
+    if err ~= nil then
+        return err
+    end
+
+    -- load the header from handle
+    local header, err = xdb.load_header(handle)
+    if err ~= nil then
+        closer()
+        return string.format("failed to load header: %s", err)
+    end
+
+    -- get the runtime ptr bytes
+    local runtime_ptr_bytes = 0
+    if header.version == xdb_structure_20 then
+        runtime_ptr_bytes = 4
+    elseif header.version == xdb_structure_30 then
+        runtime_ptr_bytes = header.runtime_ptr_bytes
+    else
+        closer()
+        return string.format("invalid structure version %d", header.version);
+    end
+
+    -- 1, confirm the xdb file size
+    -- to ensure that the maximum file pointer does not overflow
+    local max_file_ptr = ((1 << (runtime_ptr_bytes * 8)) & 0xFFFFFFFFFFFFFFFF) - 1
+    local _file_bytes  = handle:seek("end", 0)
+    -- print("max_file_ptr=", max_file_ptr, "_file_bytes", _file_bytes)
+    if _file_bytes > max_file_ptr then
+        closer()
+        return string.format("xdb file exceeds the maximum supported bytes: %d", max_file_ptr);
+    end
+
+    closer()
+    return nil
+end
+
 -- 
 -- parse ip string
 function split(str, sep)