From 501c3c086d8e6fb234f2682826d3b55c43d2810b Mon Sep 17 00:00:00 2001 From: lion Date: Thu, 23 Oct 2025 14:02:51 +0800 Subject: [PATCH] seperate all the sub commands impl --- maker/golang/cmd/bench.go | 125 +++++++ maker/golang/cmd/edit.go | 164 ++++++++ maker/golang/cmd/generate.go | 114 ++++++ maker/golang/cmd/process.go | 92 +++++ maker/golang/cmd/search.go | 133 +++++++ maker/golang/cmd/util.go | 143 +++++++ maker/golang/main.go | 703 +---------------------------------- 7 files changed, 781 insertions(+), 693 deletions(-) create mode 100644 maker/golang/cmd/bench.go create mode 100644 maker/golang/cmd/edit.go create mode 100644 maker/golang/cmd/generate.go create mode 100644 maker/golang/cmd/process.go create mode 100644 maker/golang/cmd/search.go create mode 100644 maker/golang/cmd/util.go diff --git a/maker/golang/cmd/bench.go b/maker/golang/cmd/bench.go new file mode 100644 index 0000000..1c17350 --- /dev/null +++ b/maker/golang/cmd/bench.go @@ -0,0 +1,125 @@ +package cmd + +import ( + "fmt" + "log/slog" + "os" + "time" + + "github.com/lionsoul2014/ip2region/maker/golang/xdb" +) + +func Bench() { + var err error + var dbFile, srcFile, ipVersion, logLevel = "", "", "", "" + var ignoreError = false + var fErr = iterateFlags(func(key string, val string) error { + switch key { + case "db": + dbFile = val + case "src": + srcFile = val + case "version": + ipVersion = val + case "log-level": + logLevel = val + case "ignore-error": + if val == "true" || val == "1" { + ignoreError = true + } else if val == "false" || val == "0" { + ignoreError = false + } else { + return fmt.Errorf("invalid value for ignore-error option, could be false/0 or true/1") + } + default: + return fmt.Errorf("undefined option '%s=%s'", key, val) + } + return nil + }) + if fErr != nil { + fmt.Printf("failed to parse flags: %s", fErr) + return + } + + if dbFile == "" || srcFile == "" { + fmt.Printf("%s bench [command options]\n", os.Args[0]) + fmt.Printf("options:\n") + fmt.Printf(" --db string ip2region binary xdb file path\n") + fmt.Printf(" --src string source ip text file path\n") + fmt.Printf(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused \n") + fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") + fmt.Printf(" --ignore-error bool keep going if bench failed\n") + return + } + + // check and define the IP version + var version *xdb.Version = nil + if len(ipVersion) < 2 { + slog.Error("please specify the ip version with flag --version, ipv4 or ipv6 ?") + return + } else if v, err := xdb.VersionFromName(ipVersion); err != nil { + slog.Error("failed to parse version name", "error", err) + return + } else { + version = v + } + + // check and apply the log level + err = applyLogLevel(logLevel) + if err != nil { + slog.Error("failed to apply log level", "error", err) + return + } + + searcher, err := xdb.NewSearcher(version, dbFile) + if err != nil { + fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err) + return + } + defer func() { + searcher.Close() + }() + + handle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600) + if err != nil { + fmt.Printf("failed to open source text file: %s\n", err) + return + } + + defer handle.Close() + + var count, errCount, tStart = 0, 0, time.Now() + slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile) + var iErr = xdb.IterateSegments(handle, nil, nil, func(seg *xdb.Segment) error { + var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) + slog.Debug("try to bench", "segment", l) + // mip := xdb.IPMiddle(seg.StartIP, seg.EndIP) + // for _, ip := range [][]byte{seg.StartIP, xdb.IPMiddle(seg.EndIP, mip), mip, xdb.IPMiddle(mip, seg.EndIP), seg.EndIP} { + for _, ip := range [][]byte{seg.StartIP, seg.EndIP} { + slog.Debug("|-try to bench", "ip", xdb.IP2String(ip)) + r, _, err := searcher.Search(ip) + if err != nil { + return fmt.Errorf("failed to search ip '%s': %s", xdb.IP2Long(ip), err) + } + + // check the region info + count++ + if r != seg.Region { + errCount++ + slog.Error(" --[Failed] region not match", "src", r, "dst", seg.Region) + if !ignoreError { + return fmt.Errorf("") + } + } else { + slog.Debug(" --[Ok]") + } + } + return nil + }) + if iErr != nil { + fmt.Printf("%s", err) + return + } + + slog.Info("Bench finished", "count", count, "failed", errCount, "elapsed", time.Since(tStart)) +} diff --git a/maker/golang/cmd/edit.go b/maker/golang/cmd/edit.go new file mode 100644 index 0000000..dd88966 --- /dev/null +++ b/maker/golang/cmd/edit.go @@ -0,0 +1,164 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "bufio" + "fmt" + "log/slog" + "os" + "regexp" + "strings" + "time" + + "github.com/lionsoul2014/ip2region/maker/golang/xdb" +) + +// source ip data editor + +func Edit() { + var err error + var srcFile, ipVersion = "", "" + var fErr = iterateFlags(func(key string, val string) error { + switch key { + case "src": + srcFile = val + case "version": + ipVersion = val + default: + return fmt.Errorf("undefined option '%s=%s'", key, val) + } + return nil + }) + if fErr != nil { + fmt.Printf("failed to parse flags: %s", fErr) + return + } + + if srcFile == "" { + fmt.Printf("%s edit [command options]\n", os.Args[0]) + fmt.Printf("options:\n") + fmt.Printf(" --src string source ip text file path\n") + fmt.Printf(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused \n") + return + } + + // check and define the IP version + var version *xdb.Version = nil + if len(ipVersion) < 2 { + slog.Error("please specify the ip version with flag --version, ipv4 or ipv6 ?") + return + } else if v, err := xdb.VersionFromName(ipVersion); err != nil { + slog.Error("failed to parse version name", "error", err) + return + } else { + version = v + } + + rExp, err := regexp.Compile(`\s+`) + if err != nil { + fmt.Printf("failed to compile regexp: %s\n", err) + return + } + + fmt.Printf("init the editor from source @ `%s` ... \n", srcFile) + var tStart = time.Now() + editor, err := xdb.NewEditor(version, srcFile) + if err != nil { + fmt.Printf("failed to init editor: %s", err) + return + } + + fmt.Printf("all segments loaded, length: %d, elapsed: %s\n", editor.SegLen(), time.Since(tStart)) + var help = func() { + fmt.Printf("command list: \n") + fmt.Printf(" put [segment] : put the specifield $segment\n") + fmt.Printf(" put_file [file] : put all the segments from the specified $file\n") + fmt.Printf(" list [offset] [size] : list the first $size segments start from $offset\n") + fmt.Printf(" save : save all the changes to the destination source file\n") + fmt.Printf(" quit : exit the program\n") + fmt.Printf(" help : print this help menu\n") + } + + help() + var sTip = "" + var reader = bufio.NewReader(os.Stdin) + for { + if editor.NeedSave() { + sTip = "*" + } else { + sTip = "" + } + + fmt.Printf("%seditor>> ", sTip) + line, err := reader.ReadString('\n') + if err != nil { + fmt.Printf("failed to read line from cli: %s\n", err) + break + } + + cmd := strings.TrimSpace(line) + if cmd == "help" { + help() + } else if cmd == "quit" { + if editor.NeedSave() { + fmt.Printf("there are changes that need to save, type 'quit!' to force quit\n") + } else { + break + } + } else if cmd == "quit!" { + // quit directly + break + } else if cmd == "save" { + err = editor.Save() + if err != nil { + fmt.Printf("failed to save the changes: %s\n", err) + continue + } + fmt.Printf("all segments saved to %s\n", srcFile) + } else if strings.HasPrefix(cmd, "list") { + var sErr error + off, size, l := 0, 10, len("list") + str := strings.TrimSpace(cmd) + if len(str) > l { + sets := rExp.Split(cmd, 3) + switch len(sets) { + case 2: + _, sErr = fmt.Sscanf(cmd, "%s %d", &str, &off) + case 3: + _, sErr = fmt.Sscanf(cmd, "%s %d %d", &str, &off, &size) + } + } + + if sErr != nil { + fmt.Printf("failed to parse the offset and size: %s\n", sErr) + continue + } + + fmt.Printf("+-slice(%d,%d): \n", off, size) + for _, s := range editor.Slice(off, size) { + fmt.Printf("%s\n", s) + } + } else if strings.HasPrefix(cmd, "put ") { + seg := strings.TrimSpace(cmd[len("put "):]) + o, n, err := editor.Put(seg) + if err != nil { + fmt.Printf("failed to Put(%s): %s\n", seg, err) + continue + } + fmt.Printf("Put(%s): Ok, with %d deletes and %d additions\n", seg, o, n) + } else if strings.HasPrefix(cmd, "put_file ") { + file := strings.TrimSpace(cmd[len("put_file "):]) + o, n, err := editor.PutFile(file) + if err != nil { + fmt.Printf("failed to PutFile(%s): %s\n", file, err) + continue + } + fmt.Printf("PutFile(%s): Ok, with %d deletes and %d additions\n", file, o, n) + } else if len(cmd) > 0 { + help() + } + } +} diff --git a/maker/golang/cmd/generate.go b/maker/golang/cmd/generate.go new file mode 100644 index 0000000..6916fec --- /dev/null +++ b/maker/golang/cmd/generate.go @@ -0,0 +1,114 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "fmt" + "log/slog" + "os" + "time" + + "github.com/lionsoul2014/ip2region/maker/golang/xdb" +) + +// script to do the xdb generate + +func Generate() { + var err error + var srcFile, dstFile = "", "" + var ipVersion, fieldList, logLevel = "", "", "info" + var indexPolicy = xdb.VectorIndexPolicy + var fErr = iterateFlags(func(key string, val string) error { + switch key { + case "src": + srcFile = val + case "dst": + dstFile = val + case "version": + ipVersion = val + case "log-level": + logLevel = val + case "field-list": + fieldList = val + case "index": + indexPolicy, err = xdb.IndexPolicyFromString(val) + if err != nil { + return fmt.Errorf("parse policy: %w", err) + } + default: + return fmt.Errorf("undefine option `%s=%s`", key, val) + } + + return nil + }) + if fErr != nil { + fmt.Printf("failed to parse flags: %s", fErr) + return + } + + if srcFile == "" || dstFile == "" { + fmt.Printf("%s gen [command options]\n", os.Args[0]) + fmt.Printf("options:\n") + fmt.Printf(" --src string source ip text file path\n") + fmt.Printf(" --dst string destination binary xdb file path\n") + fmt.Printf(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused \n") + fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n") + fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") + return + } + + // check and define the IP version + var version *xdb.Version = nil + if len(ipVersion) < 2 { + slog.Error("please specify the ip version with flag --version, ipv4 or ipv6 ?") + return + } else if v, err := xdb.VersionFromName(ipVersion); err != nil { + slog.Error("failed to parse version name", "error", err) + return + } else { + version = v + } + + // check and apply the log level + err = applyLogLevel(logLevel) + if err != nil { + slog.Error("failed to apply log level", "error", err) + return + } + + fields, err := getFilterFields(fieldList) + if err != nil { + slog.Error("failed to get filter fields", "error", err) + return + } + + // make the binary file + tStart := time.Now() + maker, err := xdb.NewMaker(version, indexPolicy, srcFile, dstFile, fields) + if err != nil { + fmt.Printf("failed to create %s\n", err) + return + } + + slog.Info("Generating xdb with", "src", srcFile, "dst", dstFile, "logLevel", logLevel) + err = maker.Init() + if err != nil { + fmt.Printf("failed Init: %s\n", err) + return + } + + err = maker.Start() + if err != nil { + fmt.Printf("failed Start: %s\n", err) + return + } + + err = maker.End() + if err != nil { + fmt.Printf("failed End: %s\n", err) + } + + slog.Info("make done", "elapsed", time.Since(tStart)) +} diff --git a/maker/golang/cmd/process.go b/maker/golang/cmd/process.go new file mode 100644 index 0000000..23413ea --- /dev/null +++ b/maker/golang/cmd/process.go @@ -0,0 +1,92 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "fmt" + "log/slog" + "os" + "time" + + "github.com/lionsoul2014/ip2region/maker/golang/xdb" +) + +// source data process, sort, de-duplicate, merge + +func Process() { + var err error + var srcFile, dstFile = "", "" + var fieldList, logLevel = "", "" + var fErr = iterateFlags(func(key string, val string) error { + switch key { + case "src": + srcFile = val + case "dst": + dstFile = val + case "field-list": + fieldList = val + case "log-level": + logLevel = val + default: + return fmt.Errorf("undefined option '%s=%s'", key, val) + } + return nil + }) + if fErr != nil { + fmt.Printf("failed to parse flags: %s", fErr) + return + } + + if srcFile == "" || dstFile == "" { + fmt.Printf("%s process [command options]\n", os.Args[0]) + fmt.Printf("options:\n") + fmt.Printf(" --src string source ip text file path\n") + fmt.Printf(" --dst string target ip text file path\n") + fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n") + fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") + return + } + + // check and apply the log level + err = applyLogLevel(logLevel) + if err != nil { + slog.Error("failed to apply log level", "error", err) + return + } + + fields, err := getFilterFields(fieldList) + if err != nil { + slog.Error("failed to get filter fields", "error", err) + return + } + + // make the binary file + tStart := time.Now() + processor, err := xdb.NewProcessor(srcFile, dstFile, fields) + if err != nil { + fmt.Printf("failed to create %s\n", err) + return + } + + err = processor.Init() + if err != nil { + fmt.Printf("failed Init: %s\n", err) + return + } + + slog.Info("Processing", "src", srcFile, "dst", dstFile, "logLevel", logLevel) + err = processor.Start() + if err != nil { + fmt.Printf("failed Start: %s\n", err) + return + } + + err = processor.End() + if err != nil { + fmt.Printf("failed End: %s\n", err) + } + + slog.Info("processor done", "elapsed", time.Since(tStart)) +} diff --git a/maker/golang/cmd/search.go b/maker/golang/cmd/search.go new file mode 100644 index 0000000..a64060c --- /dev/null +++ b/maker/golang/cmd/search.go @@ -0,0 +1,133 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "bufio" + "encoding/binary" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "github.com/lionsoul2014/ip2region/maker/golang/xdb" +) + +// xdb searcher test + +func Search() { + var err error + var dbFile = "" + var fErr = iterateFlags(func(key string, val string) error { + if key == "db" { + dbFile = val + } else { + return fmt.Errorf("undefined option '%s=%s'", key, val) + } + return nil + }) + if fErr != nil { + fmt.Printf("failed to parse flags: %s", fErr) + return + } + + if dbFile == "" { + fmt.Printf("%s search [command options]\n", os.Args[0]) + fmt.Printf("options:\n") + fmt.Printf(" --db string ip2region binary xdb file path\n") + return + } + + // detect the version from the xdb header + header, err := xdb.LoadXdbHeaderFromFile(dbFile) + if err != nil { + slog.Error("failed to load xdb header", "error", err) + return + } + + var version *xdb.Version = nil + versionNo := binary.LittleEndian.Uint16(header[0:]) + if versionNo == 2 { + // old xdb file + version = xdb.IPv4 + } else if versionNo == 3 { + ipNo := int(binary.LittleEndian.Uint16(header[16:])) + if ipNo == xdb.IPv4.Id { + version = xdb.IPv4 + } else if ipNo == xdb.IPv6.Id { + version = xdb.IPv6 + } else { + slog.Error("invalid ip version", "id", ipNo) + return + } + } else { + slog.Error("invalid xdb version", "versionNo", versionNo, "xdbFile", dbFile) + return + } + + searcher, err := xdb.NewSearcher(version, dbFile) + if err != nil { + fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err.Error()) + return + } + defer func() { + searcher.Close() + fmt.Printf("test program exited, thanks for trying\n") + }() + + fmt.Printf(`ip2region xdb search test program, +source xdb: %s (%s) +commands: + loadIndex : load the vector index for search speedup. + clearIndex: clear the vector index. + quit : exit the test program +`, dbFile, version.Name) + reader := bufio.NewReader(os.Stdin) + for { + fmt.Print("ip2region>> ") + str, err := reader.ReadString('\n') + if err != nil { + slog.Error("failed to read string", "error", err) + return + } + + line := strings.TrimSpace(strings.TrimSuffix(str, "\n")) + if len(line) == 0 { + continue + } + + // command interception and execution + if line == "loadIndex" { + err = searcher.LoadVectorIndex() + if err != nil { + slog.Error("failed to load vector index", "error", err) + return + } + fmt.Printf("vector index cached\n") + continue + } else if line == "clearIndex" { + searcher.ClearVectorIndex() + fmt.Printf("vector index cleared\n") + continue + } else if line == "quit" { + break + } + + ip, err := xdb.ParseIP(line) + if err != nil { + fmt.Printf("invalid ip address `%s`\n", line) + continue + } + + tStart := time.Now() + region, ioCount, err := searcher.Search(ip) + if err != nil { + fmt.Printf("\x1b[0;31m{err:%s, iocount:%d}\x1b[0m\n", err.Error(), ioCount) + } else { + fmt.Printf("\x1b[0;32m{region:%s, iocount:%d, took:%s}\x1b[0m\n", region, ioCount, time.Since(tStart)) + } + } +} diff --git a/maker/golang/cmd/util.go b/maker/golang/cmd/util.go new file mode 100644 index 0000000..e0f360f --- /dev/null +++ b/maker/golang/cmd/util.go @@ -0,0 +1,143 @@ +package cmd + +import ( + "fmt" + "log/slog" + "os" + "regexp" + "sort" + "strconv" + "strings" +) + +func PrintHelp() { + fmt.Printf("ip2region xdb maker\n") + fmt.Printf("%s [command] [command options]\n", os.Args[0]) + fmt.Printf("Command: \n") + fmt.Printf(" gen generate the binary xdb file\n") + fmt.Printf(" search binary xdb search test\n") + fmt.Printf(" bench binary xdb bench test\n") + fmt.Printf(" edit edit the source ip data\n") + fmt.Printf(" process process the source ip data\n") +} + +// Iterate the cli flags +func iterateFlags(cb func(key string, val string) error) error { + for i := 2; i < len(os.Args); i++ { + r := os.Args[i] + if len(r) < 5 { + continue + } + + if strings.Index(r, "--") != 0 { + continue + } + + var sIdx = strings.Index(r, "=") + if sIdx < 0 { + return fmt.Errorf("missing = for args pair '%s'", r) + } + + if err := cb(r[2:sIdx], r[sIdx+1:]); err != nil { + return err + } + } + + return nil +} + +func applyLogLevel(logLevel string) error { + // check and apply the log level + var levelLog = slog.LevelInfo + switch strings.ToLower(logLevel) { + case "debug": + levelLog = slog.LevelDebug + case "info": + levelLog = slog.LevelInfo + case "warn": + levelLog = slog.LevelWarn + case "error": + levelLog = slog.LevelError + case "": + // ignore the empty value + // and default it to LevelInfo + default: + return fmt.Errorf("invalid log level %s", logLevel) + } + + slog.SetLogLoggerLevel(levelLog) + return nil +} + +var pattern = regexp.MustCompile(`^(\d+(-\d+)?)$`) + +func getFilterFields(fieldList string) ([]int, error) { + if len(fieldList) == 0 { + return []int{}, nil + } + + var fields []int + var mapping = make(map[string]string) + fList := strings.Split(fieldList, ",") + for _, f := range fList { + f = strings.TrimSpace(f) + if len(f) == 0 { + return nil, fmt.Errorf("empty field index value `%s`", f) + } + + ms := pattern.FindString(f) + if len(ms) == 0 { + return nil, fmt.Errorf("field `%s` is not a number or number range", f) + } + + // if strings.Index(ms, "-") == -1 { + if !strings.Contains(ms, "-") { + if _, ok := mapping[ms]; ok { + return nil, fmt.Errorf("duplicate option `%s`", f) + } + + idx, err := strconv.Atoi(ms) + if err != nil { + return nil, fmt.Errorf("field index `%s` not an integer", f) + } + + mapping[ms] = ms + fields = append(fields, idx) + continue + } + + ra := strings.Split(ms, "-") + if len(ra) != 2 { + return nil, fmt.Errorf("invalid field index range `%s`", ms) + } + + start, err := strconv.Atoi(ra[0]) + if err != nil { + return nil, fmt.Errorf("range start `%s` not an integer", ra[0]) + } + + end, err := strconv.Atoi(ra[1]) + if err != nil { + return nil, fmt.Errorf("range end `%s` not an integer", ra[1]) + } + + if start > end { + return nil, fmt.Errorf("index range start(%d) should <= end(%d)", start, end) + } + + for i := start; i <= end; i++ { + s := strconv.Itoa(i) + if _, ok := mapping[s]; ok { + return nil, fmt.Errorf("duplicate option `%s`", s) + } + + mapping[s] = s + fields = append(fields, i) + } + } + + // sort the fields + sort.Ints(fields) + // fmt.Printf("%+v\n", fields) + return fields, nil +} diff --git a/maker/golang/main.go b/maker/golang/main.go index 5eea2fc..28fdd84 100644 --- a/maker/golang/main.go +++ b/maker/golang/main.go @@ -5,717 +5,34 @@ package main import ( - "bufio" - "encoding/binary" - "fmt" "log" - "log/slog" "os" - "regexp" - "sort" - "strconv" "strings" - "time" - "github.com/lionsoul2014/ip2region/maker/golang/xdb" + "github.com/lionsoul2014/ip2region/maker/golang/cmd" ) -func printHelp() { - fmt.Printf("ip2region xdb maker\n") - fmt.Printf("%s [command] [command options]\n", os.Args[0]) - fmt.Printf("Command: \n") - fmt.Printf(" gen generate the binary xdb file\n") - fmt.Printf(" search binary xdb search test\n") - fmt.Printf(" bench binary xdb bench test\n") - fmt.Printf(" edit edit the source ip data\n") - fmt.Printf(" process process the source ip data\n") -} - -// Iterate the cli flags -func iterateFlags(cb func(key string, val string) error) error { - for i := 2; i < len(os.Args); i++ { - r := os.Args[i] - if len(r) < 5 { - continue - } - - if strings.Index(r, "--") != 0 { - continue - } - - var sIdx = strings.Index(r, "=") - if sIdx < 0 { - return fmt.Errorf("missing = for args pair '%s'", r) - } - - if err := cb(r[2:sIdx], r[sIdx+1:]); err != nil { - return err - } - } - - return nil -} - -func applyLogLevel(logLevel string) error { - // check and apply the log level - var levelLog = slog.LevelInfo - switch strings.ToLower(logLevel) { - case "debug": - levelLog = slog.LevelDebug - case "info": - levelLog = slog.LevelInfo - case "warn": - levelLog = slog.LevelWarn - case "error": - levelLog = slog.LevelError - case "": - // ignore the empty value - // and default it to LevelInfo - default: - return fmt.Errorf("invalid log level %s", logLevel) - } - - slog.SetLogLoggerLevel(levelLog) - return nil -} - -var pattern = regexp.MustCompile("^(\\d+(-\\d+)?)$") - -func getFilterFields(fieldList string) ([]int, error) { - if len(fieldList) == 0 { - return []int{}, nil - } - - var fields []int - var mapping = make(map[string]string) - fList := strings.Split(fieldList, ",") - for _, f := range fList { - f = strings.TrimSpace(f) - if len(f) == 0 { - return nil, fmt.Errorf("empty field index value `%s`", f) - } - - ms := pattern.FindString(f) - if len(ms) == 0 { - return nil, fmt.Errorf("field `%s` is not a number or number range", f) - } - - if strings.Index(ms, "-") == -1 { - if _, ok := mapping[ms]; ok { - return nil, fmt.Errorf("duplicate option `%s`", f) - } - - idx, err := strconv.Atoi(ms) - if err != nil { - return nil, fmt.Errorf("field index `%s` not an integer", f) - } - - mapping[ms] = ms - fields = append(fields, idx) - continue - } - - ra := strings.Split(ms, "-") - if len(ra) != 2 { - return nil, fmt.Errorf("invalid field index range `%s`", ms) - } - - start, err := strconv.Atoi(ra[0]) - if err != nil { - return nil, fmt.Errorf("range start `%s` not an integer", ra[0]) - } - - end, err := strconv.Atoi(ra[1]) - if err != nil { - return nil, fmt.Errorf("range end `%s` not an integer", ra[1]) - } - - if start > end { - return nil, fmt.Errorf("index range start(%d) should <= end(%d)", start, end) - } - - for i := start; i <= end; i++ { - s := strconv.Itoa(i) - if _, ok := mapping[s]; ok { - return nil, fmt.Errorf("duplicate option `%s`", s) - } - - mapping[s] = s - fields = append(fields, i) - } - } - - // sort the fields - sort.Ints(fields) - // fmt.Printf("%+v\n", fields) - return fields, nil -} - -func genDb() { - var err error - var srcFile, dstFile = "", "" - var ipVersion, fieldList, logLevel = "", "", "info" - var indexPolicy = xdb.VectorIndexPolicy - var fErr = iterateFlags(func(key string, val string) error { - switch key { - case "src": - srcFile = val - case "dst": - dstFile = val - case "version": - ipVersion = val - case "log-level": - logLevel = val - case "field-list": - fieldList = val - case "index": - indexPolicy, err = xdb.IndexPolicyFromString(val) - if err != nil { - return fmt.Errorf("parse policy: %w", err) - } - default: - return fmt.Errorf("undefine option `%s=%s`\n", key, val) - } - - return nil - }) - if fErr != nil { - fmt.Printf("failed to parse flags: %s", fErr) - return - } - - if srcFile == "" || dstFile == "" { - fmt.Printf("%s gen [command options]\n", os.Args[0]) - fmt.Printf("options:\n") - fmt.Printf(" --src string source ip text file path\n") - fmt.Printf(" --dst string destination binary xdb file path\n") - fmt.Printf(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused \n") - fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n") - fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") - return - } - - // check and define the IP version - var version *xdb.Version = nil - if len(ipVersion) < 2 { - slog.Error("please specify the ip version with flag --version, ipv4 or ipv6 ?") - return - } else if v, err := xdb.VersionFromName(ipVersion); err != nil { - slog.Error("failed to parse version name", "error", err) - return - } else { - version = v - } - - // check and apply the log level - err = applyLogLevel(logLevel) - if err != nil { - slog.Error("failed to apply log level", "error", err) - return - } - - fields, err := getFilterFields(fieldList) - if err != nil { - slog.Error("failed to get filter fields", "error", err) - return - } - - // make the binary file - tStart := time.Now() - maker, err := xdb.NewMaker(version, indexPolicy, srcFile, dstFile, fields) - if err != nil { - fmt.Printf("failed to create %s\n", err) - return - } - - slog.Info("Generating xdb with", "src", srcFile, "dst", dstFile, "logLevel", logLevel) - err = maker.Init() - if err != nil { - fmt.Printf("failed Init: %s\n", err) - return - } - - err = maker.Start() - if err != nil { - fmt.Printf("failed Start: %s\n", err) - return - } - - err = maker.End() - if err != nil { - fmt.Printf("failed End: %s\n", err) - } - - slog.Info("make done", "elapsed", time.Since(tStart)) -} - -func testSearch() { - var err error - var dbFile = "" - var fErr = iterateFlags(func(key string, val string) error { - if key == "db" { - dbFile = val - } else { - return fmt.Errorf("undefined option '%s=%s'\n", key, val) - } - return nil - }) - if fErr != nil { - fmt.Printf("failed to parse flags: %s", fErr) - return - } - - if dbFile == "" { - fmt.Printf("%s search [command options]\n", os.Args[0]) - fmt.Printf("options:\n") - fmt.Printf(" --db string ip2region binary xdb file path\n") - return - } - - // detect the version from the xdb header - header, err := xdb.LoadXdbHeaderFromFile(dbFile) - if err != nil { - slog.Error("failed to load xdb header", "error", err) - return - } - - var version *xdb.Version = nil - versionNo := binary.LittleEndian.Uint16(header[0:]) - if versionNo == 2 { - // old xdb file - version = xdb.IPv4 - } else if versionNo == 3 { - ipNo := int(binary.LittleEndian.Uint16(header[16:])) - if ipNo == xdb.IPv4.Id { - version = xdb.IPv4 - } else if ipNo == xdb.IPv6.Id { - version = xdb.IPv6 - } else { - slog.Error("invalid ip version", "id", ipNo) - return - } - } else { - slog.Error("invalid xdb version", "versionNo", versionNo, "xdbFile", dbFile) - return - } - - searcher, err := xdb.NewSearcher(version, dbFile) - if err != nil { - fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err.Error()) - return - } - defer func() { - searcher.Close() - fmt.Printf("test program exited, thanks for trying\n") - }() - - fmt.Printf(`ip2region xdb search test program, -source xdb: %s (%s) -commands: - loadIndex : load the vector index for search speedup. - clearIndex: clear the vector index. - quit : exit the test program -`, dbFile, version.Name) - reader := bufio.NewReader(os.Stdin) - for { - fmt.Print("ip2region>> ") - str, err := reader.ReadString('\n') - if err != nil { - slog.Error("failed to read string", "error", err) - return - } - - line := strings.TrimSpace(strings.TrimSuffix(str, "\n")) - if len(line) == 0 { - continue - } - - // command interception and execution - if line == "loadIndex" { - err = searcher.LoadVectorIndex() - if err != nil { - slog.Error("failed to load vector index", "error", err) - return - } - fmt.Printf("vector index cached\n") - continue - } else if line == "clearIndex" { - searcher.ClearVectorIndex() - fmt.Printf("vector index cleared\n") - continue - } else if line == "quit" { - break - } - - ip, err := xdb.ParseIP(line) - if err != nil { - fmt.Printf("invalid ip address `%s`\n", line) - continue - } - - tStart := time.Now() - region, ioCount, err := searcher.Search(ip) - if err != nil { - fmt.Printf("\x1b[0;31m{err:%s, iocount:%d}\x1b[0m\n", err.Error(), ioCount) - } else { - fmt.Printf("\x1b[0;32m{region:%s, iocount:%d, took:%s}\x1b[0m\n", region, ioCount, time.Since(tStart)) - } - } -} - -func testBench() { - var err error - var dbFile, srcFile, ipVersion, logLevel = "", "", "", "" - var ignoreError = false - var fErr = iterateFlags(func(key string, val string) error { - switch key { - case "db": - dbFile = val - case "src": - srcFile = val - case "version": - ipVersion = val - case "log-level": - logLevel = val - case "ignore-error": - if val == "true" || val == "1" { - ignoreError = true - } else if val == "false" || val == "0" { - ignoreError = false - } else { - return fmt.Errorf("invalid value for ignore-error option, could be false/0 or true/1\n") - } - default: - return fmt.Errorf("undefined option '%s=%s'\n", key, val) - } - return nil - }) - if fErr != nil { - fmt.Printf("failed to parse flags: %s", fErr) - return - } - - if dbFile == "" || srcFile == "" { - fmt.Printf("%s bench [command options]\n", os.Args[0]) - fmt.Printf("options:\n") - fmt.Printf(" --db string ip2region binary xdb file path\n") - fmt.Printf(" --src string source ip text file path\n") - fmt.Printf(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused \n") - fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") - fmt.Printf(" --ignore-error bool keep going if bench failed\n") - return - } - - // check and define the IP version - var version *xdb.Version = nil - if len(ipVersion) < 2 { - slog.Error("please specify the ip version with flag --version, ipv4 or ipv6 ?") - return - } else if v, err := xdb.VersionFromName(ipVersion); err != nil { - slog.Error("failed to parse version name", "error", err) - return - } else { - version = v - } - - // check and apply the log level - err = applyLogLevel(logLevel) - if err != nil { - slog.Error("failed to apply log level", "error", err) - return - } - - searcher, err := xdb.NewSearcher(version, dbFile) - if err != nil { - fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err) - return - } - defer func() { - searcher.Close() - }() - - handle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600) - if err != nil { - fmt.Printf("failed to open source text file: %s\n", err) - return - } - - var count, errCount, tStart = 0, 0, time.Now() - slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile) - var iErr = xdb.IterateSegments(handle, nil, nil, func(seg *xdb.Segment) error { - var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) - slog.Debug("try to bench", "segment", l) - // mip := xdb.IPMiddle(seg.StartIP, seg.EndIP) - // for _, ip := range [][]byte{seg.StartIP, xdb.IPMiddle(seg.EndIP, mip), mip, xdb.IPMiddle(mip, seg.EndIP), seg.EndIP} { - for _, ip := range [][]byte{seg.StartIP, seg.EndIP} { - slog.Debug("|-try to bench", "ip", xdb.IP2String(ip)) - r, _, err := searcher.Search(ip) - if err != nil { - return fmt.Errorf("failed to search ip '%s': %s\n", xdb.IP2Long(ip), err) - } - - // check the region info - count++ - if r != seg.Region { - errCount++ - slog.Error(" --[Failed] region not match", "src", r, "dst", seg.Region) - if ignoreError == false { - return fmt.Errorf("") - } - } else { - slog.Debug(" --[Ok]") - } - } - return nil - }) - if iErr != nil { - fmt.Printf("%s", err) - return - } - - slog.Info("Bench finished", "count", count, "failed", errCount, "elapsed", time.Since(tStart)) -} - -func edit() { - var err error - var srcFile, ipVersion = "", "" - var fErr = iterateFlags(func(key string, val string) error { - switch key { - case "src": - srcFile = val - case "version": - ipVersion = val - default: - return fmt.Errorf("undefined option '%s=%s'\n", key, val) - } - return nil - }) - if fErr != nil { - fmt.Printf("failed to parse flags: %s", fErr) - return - } - - if srcFile == "" { - fmt.Printf("%s edit [command options]\n", os.Args[0]) - fmt.Printf("options:\n") - fmt.Printf(" --src string source ip text file path\n") - fmt.Printf(" --version string IP version, options: ipv4/ipv6, specify this flag so you don't get confused \n") - return - } - - // check and define the IP version - var version *xdb.Version = nil - if len(ipVersion) < 2 { - slog.Error("please specify the ip version with flag --version, ipv4 or ipv6 ?") - return - } else if v, err := xdb.VersionFromName(ipVersion); err != nil { - slog.Error("failed to parse version name", "error", err) - return - } else { - version = v - } - - rExp, err := regexp.Compile("\\s+") - if err != nil { - fmt.Printf("failed to compile regexp: %s\n", err) - return - } - - fmt.Printf("init the editor from source @ `%s` ... \n", srcFile) - var tStart = time.Now() - editor, err := xdb.NewEditor(version, srcFile) - if err != nil { - fmt.Printf("failed to init editor: %s", err) - return - } - - fmt.Printf("all segments loaded, length: %d, elapsed: %s\n", editor.SegLen(), time.Since(tStart)) - var help = func() { - fmt.Printf("command list: \n") - fmt.Printf(" put [segment] : put the specifield $segment\n") - fmt.Printf(" put_file [file] : put all the segments from the specified $file\n") - fmt.Printf(" list [offset] [size] : list the first $size segments start from $offset\n") - fmt.Printf(" save : save all the changes to the destination source file\n") - fmt.Printf(" quit : exit the program\n") - fmt.Printf(" help : print this help menu\n") - } - - help() - var sTip = "" - var reader = bufio.NewReader(os.Stdin) - for { - if editor.NeedSave() { - sTip = "*" - } else { - sTip = "" - } - - fmt.Printf("%seditor>> ", sTip) - line, err := reader.ReadString('\n') - if err != nil { - fmt.Printf("failed to read line from cli: %s\n", err) - break - } - - cmd := strings.TrimSpace(line) - if cmd == "help" { - help() - } else if cmd == "quit" { - if editor.NeedSave() { - fmt.Printf("there are changes that need to save, type 'quit!' to force quit\n") - } else { - break - } - } else if cmd == "quit!" { - // quit directly - break - } else if cmd == "save" { - err = editor.Save() - if err != nil { - fmt.Printf("failed to save the changes: %s\n", err) - continue - } - fmt.Printf("all segments saved to %s\n", srcFile) - } else if strings.HasPrefix(cmd, "list") { - var sErr error - off, size, l := 0, 10, len("list") - str := strings.TrimSpace(cmd) - if len(str) > l { - sets := rExp.Split(cmd, 3) - switch len(sets) { - case 2: - _, sErr = fmt.Sscanf(cmd, "%s %d", &str, &off) - case 3: - _, sErr = fmt.Sscanf(cmd, "%s %d %d", &str, &off, &size) - } - } - - if sErr != nil { - fmt.Printf("failed to parse the offset and size: %s\n", sErr) - continue - } - - fmt.Printf("+-slice(%d,%d): \n", off, size) - for _, s := range editor.Slice(off, size) { - fmt.Printf("%s\n", s) - } - } else if strings.HasPrefix(cmd, "put ") { - seg := strings.TrimSpace(cmd[len("put "):]) - o, n, err := editor.Put(seg) - if err != nil { - fmt.Printf("failed to Put(%s): %s\n", seg, err) - continue - } - fmt.Printf("Put(%s): Ok, with %d deletes and %d additions\n", seg, o, n) - } else if strings.HasPrefix(cmd, "put_file ") { - file := strings.TrimSpace(cmd[len("put_file "):]) - o, n, err := editor.PutFile(file) - if err != nil { - fmt.Printf("failed to PutFile(%s): %s\n", file, err) - continue - } - fmt.Printf("PutFile(%s): Ok, with %d deletes and %d additions\n", file, o, n) - } else if len(cmd) > 0 { - help() - } - } -} - -func process() { - var err error - var srcFile, dstFile = "", "" - var fieldList, logLevel = "", "" - var fErr = iterateFlags(func(key string, val string) error { - switch key { - case "src": - srcFile = val - case "dst": - dstFile = val - case "field-list": - fieldList = val - case "log-level": - logLevel = val - default: - return fmt.Errorf("undefined option '%s=%s'\n", key, val) - } - return nil - }) - if fErr != nil { - fmt.Printf("failed to parse flags: %s", fErr) - return - } - - if srcFile == "" || dstFile == "" { - fmt.Printf("%s process [command options]\n", os.Args[0]) - fmt.Printf("options:\n") - fmt.Printf(" --src string source ip text file path\n") - fmt.Printf(" --dst string target ip text file path\n") - fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n") - fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") - return - } - - // check and apply the log level - err = applyLogLevel(logLevel) - if err != nil { - slog.Error("failed to apply log level", "error", err) - return - } - - fields, err := getFilterFields(fieldList) - if err != nil { - slog.Error("failed to get filter fields", "error", err) - return - } - - // make the binary file - tStart := time.Now() - processor, err := xdb.NewProcessor(srcFile, dstFile, fields) - if err != nil { - fmt.Printf("failed to create %s\n", err) - return - } - - err = processor.Init() - if err != nil { - fmt.Printf("failed Init: %s\n", err) - return - } - - slog.Info("Processing", "src", srcFile, "dst", dstFile, "logLevel", logLevel) - err = processor.Start() - if err != nil { - fmt.Printf("failed Start: %s\n", err) - return - } - - err = processor.End() - if err != nil { - fmt.Printf("failed End: %s\n", err) - } - - slog.Info("processor done", "elapsed", time.Since(tStart)) -} - func main() { if len(os.Args) < 2 { - printHelp() + cmd.PrintHelp() return } log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) switch strings.ToLower(os.Args[1]) { case "gen": - genDb() + cmd.Generate() case "search": - testSearch() + cmd.Search() case "bench": - testBench() + cmd.Bench() case "edit": - edit() + cmd.Edit() case "process": - process() + cmd.Process() + case "stat": + cmd.Stats() default: - printHelp() + cmd.PrintHelp() } }