From c0058781c9c415ec846e9140f87f30305fd80b98 Mon Sep 17 00:00:00 2001 From: lion Date: Thu, 4 Sep 2025 11:14:58 +0800 Subject: [PATCH] version expected for xdb gen and auto version detect for search test --- maker/golang/main.go | 69 ++++++++++++++++++++++++------------ maker/golang/xdb/maker.go | 7 +++- maker/golang/xdb/searcher.go | 44 +++++++++++++++++++++++ maker/golang/xdb/util.go | 4 +++ maker/golang/xdb/version.go | 8 ++++- 5 files changed, 107 insertions(+), 25 deletions(-) diff --git a/maker/golang/main.go b/maker/golang/main.go index 764575e..0a71198 100644 --- a/maker/golang/main.go +++ b/maker/golang/main.go @@ -6,6 +6,7 @@ package main import ( "bufio" + "encoding/binary" "fmt" "log" "log/slog" @@ -23,7 +24,7 @@ func printHelp() { fmt.Printf("ip2region xdb maker\n") fmt.Printf("%s [command] [command options]\n", os.Args[0]) fmt.Printf("Command: \n") - fmt.Printf(" gen generate the binary db file\n") + fmt.Printf(" gen generate the binary xdb file\n") fmt.Printf(" search binary xdb search test\n") fmt.Printf(" bench binary xdb bench test\n") fmt.Printf(" edit edit the source ip data\n") @@ -152,7 +153,7 @@ func getFilterFields(fieldList string) ([]int, error) { func genDb() { var err error var srcFile, dstFile = "", "" - var ipVersion, fieldList, logLevel = "ipv4", "", "info" + var ipVersion, fieldList, logLevel = "", "", "info" var indexPolicy = xdb.VectorIndexPolicy var fErr = iterateFlags(func(key string, val string) error { switch key { @@ -187,12 +188,23 @@ func genDb() { fmt.Printf("options:\n") fmt.Printf(" --src string source ip text file path\n") fmt.Printf(" --dst string destination binary xdb file path\n") - fmt.Printf(" --version string IP version, options: ipv4/ipv6, default to ipv4\n") + fmt.Printf(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused \n") fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n") fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") return } + // check and define the IP version + var version *xdb.Version = nil + if len(ipVersion) < 2 { + slog.Error("please specify the ip version with flag --version, ipv4 or ipv6 ?") + return + } else if v, err := xdb.VersionFromName(ipVersion); err != nil { + slog.Error("failed to parse version name", "error", err) + } else { + version = v + } + // check and apply the log level err = applyLogLevel(logLevel) if err != nil { @@ -206,14 +218,6 @@ func genDb() { return } - // check and define the IP version - var version *xdb.Version = nil - if v, err := xdb.VersionFromName(ipVersion); err != nil { - slog.Error("failed to parse version name", "error", err) - } else { - version = v - } - // make the binary file tStart := time.Now() maker, err := xdb.NewMaker(version, indexPolicy, srcFile, dstFile, fields) @@ -245,12 +249,10 @@ func genDb() { func testSearch() { var err error - var dbFile, ipVersion = "", "v4" + var dbFile = "" var fErr = iterateFlags(func(key string, val string) error { if key == "db" { dbFile = val - } else if key == "version" { - ipVersion = val } else { return fmt.Errorf("undefined option '%s=%s'\n", key, val) } @@ -265,16 +267,34 @@ func testSearch() { fmt.Printf("%s search [command options]\n", os.Args[0]) fmt.Printf("options:\n") fmt.Printf(" --db string ip2region binary xdb file path\n") - fmt.Printf(" --version string IP version, options: ipv4/ipv6\n") return } - // check and parse the IP version + // detect the version from the xdb header + header, err := xdb.LoadXdbHeaderFromFile(dbFile) + if err != nil { + slog.Error("failed to load xdb header", "error", err) + return + } + var version *xdb.Version = nil - if v, err := xdb.VersionFromName(ipVersion); err != nil { - slog.Error("failed to parse version name", "error", err) + versionNo := binary.LittleEndian.Uint16(header[0:]) + if versionNo == 2 { + // old xdb file + version = xdb.V4 + } else if versionNo == 3 { + ipNo := int(binary.LittleEndian.Uint16(header[16:])) + if ipNo == xdb.V4.Id { + version = xdb.V4 + } else if ipNo == xdb.V6.Id { + version = xdb.V6 + } else { + slog.Error("invalid ip version", "id", ipNo) + return + } } else { - version = v + slog.Error("invalid xdb version", "versionNo", versionNo, "xdbFile", dbFile) + return } searcher, err := xdb.NewSearcher(version, dbFile) @@ -287,10 +307,13 @@ func testSearch() { fmt.Printf("test program exited, thanks for trying\n") }() - fmt.Println(`ip2region xdb search test program, commands: -loadIndex : load the vector index for search speedup. -clearIndex: clear the vector index. -quit : exit the test program`) + fmt.Printf(`ip2region xdb search test program, +source xdb: %s (%s) +commands: + loadIndex : load the vector index for search speedup. + clearIndex: clear the vector index. + quit : exit the test program +`, dbFile, version.Name) reader := bufio.NewReader(os.Stdin) for { fmt.Print("ip2region>> ") diff --git a/maker/golang/xdb/maker.go b/maker/golang/xdb/maker.go index ae90ff0..aef78f1 100644 --- a/maker/golang/xdb/maker.go +++ b/maker/golang/xdb/maker.go @@ -60,7 +60,7 @@ import ( ) const ( - VersionNo = 3 + VersionNo = 3 // since 2025/09/01 (IPv6 supporting) HeaderInfoLength = 256 VectorIndexRows = 256 VectorIndexCols = 256 @@ -183,6 +183,11 @@ func (m *Maker) loadSegments() error { var iErr = IterateSegments(m.srcHandle, func(l string) { slog.Debug("loaded", "segment", l) }, func(seg *Segment) error { + // ip version check + if len(seg.StartIP) != m.version.Bytes { + return fmt.Errorf("invalid ip segment(%s expected)", m.version.Name) + } + // check the continuity of the data segment if err := seg.AfterCheck(last); err != nil { return err diff --git a/maker/golang/xdb/searcher.go b/maker/golang/xdb/searcher.go index 9fcc743..986e5a5 100644 --- a/maker/golang/xdb/searcher.go +++ b/maker/golang/xdb/searcher.go @@ -90,6 +90,11 @@ func (s *Searcher) ClearVectorIndex() { // Search find the region for the specified ip address func (s *Searcher) Search(ip []byte) (string, int, error) { + // version check + if len(ip) != s.version.Bytes { + return "", 0, fmt.Errorf("invalid ip address(%s expected)", s.version.Name) + } + // locate the segment index block based on the vector index var ioCount = 0 var il0, il1, bytes, tBytes = int(ip[0]), int(ip[1]), len(ip), len(ip) << 1 @@ -180,3 +185,42 @@ func (s *Searcher) Search(ip []byte) (string, int, error) { return string(regionBuff), ioCount, nil } + +// LoadXdbHeader load the header info from the specified handle +func LoadXdbHeader(handle *os.File) ([]byte, error) { + _, err := handle.Seek(0, 0) + if err != nil { + return nil, fmt.Errorf("seek to the header: %w", err) + } + + var buff = make([]byte, HeaderInfoLength) + rLen, err := handle.Read(buff) + if err != nil { + return nil, err + } + + if rLen != len(buff) { + return nil, fmt.Errorf("incomplete read: readed bytes should be %d", len(buff)) + } + + return buff, nil +} + +// LoadXdbHeaderFromFile load header info from the specified db file path +func LoadXdbHeaderFromFile(dbFile string) ([]byte, error) { + handle, err := os.OpenFile(dbFile, os.O_RDONLY, 0600) + if err != nil { + return nil, fmt.Errorf("open xdb file `%s`: %w", dbFile, err) + } + + defer func(handle *os.File) { + _ = handle.Close() + }(handle) + + header, err := LoadXdbHeader(handle) + if err != nil { + return nil, err + } + + return header, nil +} diff --git a/maker/golang/xdb/util.go b/maker/golang/xdb/util.go index dcd344e..74014d7 100644 --- a/maker/golang/xdb/util.go +++ b/maker/golang/xdb/util.go @@ -132,6 +132,10 @@ func IterateSegments(handle *os.File, before func(l string), cb func(seg *Segmen return fmt.Errorf("check end ip `%s`: %s", ps[1], err) } + if len(sip) != len(eip) { + return fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l) + } + if IPCompare(sip, eip) > 0 { return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) } diff --git a/maker/golang/xdb/version.go b/maker/golang/xdb/version.go index 51e89c9..0465fce 100644 --- a/maker/golang/xdb/version.go +++ b/maker/golang/xdb/version.go @@ -11,6 +11,8 @@ import ( type Version struct { Id int + Name string + Bytes int SegmentIndexSize int } @@ -18,15 +20,19 @@ var ( VX = &Version{} V4 = &Version{ Id: 4, + Name: "IPv4", + Bytes: 4, SegmentIndexSize: 14, // 4 + 4 + 2 + 4 } V6 = &Version{ Id: 6, + Name: "IPv6", + Bytes: 16, SegmentIndexSize: 38, // 16 + 16 + 2 + 4 } ) -func VersionFromData(ip string) (*Version, error) { +func VersionFromIP(ip string) (*Version, error) { bytes, err := ParseIP(ip) if err != nil { return VX, fmt.Errorf("parse ip fail: %w", err)