archive-gh-me/ip2region
1
0
Fork 0
mirror of https://github.com/lionsoul2014/ip2region.git synced 2025-12-08 19:25:22 +00:00
Code Issues Projects Releases Wiki Activity

xdb ipv6 supporting (bytes operation is working)

Browse Source
This commit is contained in:
lion 2025-09-03 17:16:05 +08:00
parent 4996c0ff6a
commit b73e190905
9 changed files with 406 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
data/segments.tests.mixed Normal file
View File

@ -0,0 +1,10 @@
192.168.2.1|192.168.2.20|0|0|0|内网IP|办公室A
192.168.2.21|192.168.2.30|0|0|0|内网IP|办公室A
192.168.2.31|192.168.2.60|0|0|0|内网IP|办公室B
192.168.2.61|192.168.2.91|0|0|0|内网IP|办公室B
223.255.236.0|223.255.239.255|中国|0|上海|上海市|电信
2c0f:fff1::|2c0f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|毛里求斯|威廉平原区||卡特勒博尔纳||专线用户
2e00::|2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|德国|黑森||美因河畔法兰克福||专线用户
3000::|fbff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||||专线用户|IANA
fe00::|fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||||专线用户|IANA
fe80::|febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||||专线用户|IANA

35
maker/golang/main.go
View File

@ -7,7 +7,6 @@ package main
import (
"bufio"
"fmt"
"github.com/lionsoul2014/ip2region/maker/golang/xdb"
"log"
"log/slog"
"os"
@ -16,6 +15,8 @@ import (
"strconv"
"strings"
"time"
"github.com/lionsoul2014/ip2region/maker/golang/xdb"
)
func printHelp() {
@ -151,7 +152,7 @@ func getFilterFields(fieldList string) ([]int, error) {
func genDb() {
var err error
var srcFile, dstFile = "", ""
var fieldList, logLevel = "", "info"
var ipVersion, fieldList, logLevel = "auto", "", "info"
var indexPolicy = xdb.VectorIndexPolicy
var fErr = iterateFlags(func(key string, val string) error {
switch key {
@ -159,6 +160,8 @@ func genDb() {
srcFile = val
case "dst":
dstFile = val
case "version":
ipVersion = val
case "log-level":
logLevel = val
case "field-list":
@ -184,6 +187,7 @@ func genDb() {
fmt.Printf("options:\n")
fmt.Printf(" --src string source ip text file path\n")
fmt.Printf(" --dst string destination binary xdb file path\n")
fmt.Printf(" --version string IP version, options: ipv4/ipv6, default auto (detect from the source file)\n")
fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n")
fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n")
return
@ -202,9 +206,19 @@ func genDb() {
return
}
// check and define the IP version
var version *xdb.Version = nil
if ipVersion == "auto" {
version = xdb.V4
} else if v, err := xdb.VersionFromName(ipVersion); err != nil {
slog.Error("failed to parse version name", "error", err)
} else {
version = v
}
// make the binary file
tStart := time.Now()
maker, err := xdb.NewMaker(indexPolicy, srcFile, dstFile, fields)
maker, err := xdb.NewMaker(version, indexPolicy, srcFile, dstFile, fields)
if err != nil {
fmt.Printf("failed to create %s\n", err)
return
@ -254,7 +268,7 @@ func testSearch() {
return
}
searcher, err := xdb.NewSearcher(dbFile)
searcher, err := xdb.NewSearcher(xdb.V4, dbFile)
if err != nil {
fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err.Error())
return
@ -299,7 +313,7 @@ quit : exit the test program`)
break
}
ip, err := xdb.CheckIP(line)
ip, err := xdb.ParseIP(line)
if err != nil {
fmt.Printf("invalid ip address `%s`\n", line)
continue
@ -362,7 +376,7 @@ func testBench() {
return
}
searcher, err := xdb.NewSearcher(dbFile)
searcher, err := xdb.NewSearcher(xdb.V4, dbFile)
if err != nil {
fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err)
return
@ -382,12 +396,13 @@ func testBench() {
var iErr = xdb.IterateSegments(handle, nil, func(seg *xdb.Segment) error {
var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region)
slog.Debug("try to bench", "segment", l)
mip := xdb.MidIP(seg.StartIP, seg.EndIP)
for _, ip := range []uint32{seg.StartIP, xdb.MidIP(seg.EndIP, mip), mip, xdb.MidIP(mip, seg.EndIP), seg.EndIP} {
slog.Debug("|-try to bench", "ip", xdb.Long2IP(ip))
mip := xdb.IPMiddle(seg.StartIP, seg.EndIP)
for _, ip := range [][]byte{
seg.StartIP, xdb.IPMiddle(seg.EndIP, mip), mip, xdb.IPMiddle(mip, seg.EndIP), seg.EndIP} {
slog.Debug("|-try to bench", "ip", xdb.IP2String(ip))
r, _, err := searcher.Search(ip)
if err != nil {
return fmt.Errorf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err)
return fmt.Errorf("failed to search ip '%s': %s\n", xdb.IP2Long(ip), err)
}
// check the region info

18
maker/golang/xdb/editor.go
View File

@ -120,11 +120,13 @@ func (e *Editor) Put(ip string) (int, int, error) {
// the following position relationships.
// 1, A - fully contained like:
// StartIP------seg.StartIP--------seg.EndIP----EndIP
// |------------------|
//
// |------------------|
//
// 2, B - intersect like:
// StartIP------seg.StartIP------EndIP------|
// |---------------------seg.EndIP
//
// |---------------------seg.EndIP
func (e *Editor) PutSegment(seg *Segment) (int, int, error) {
var next *list.Element
var eList []*list.Element
@ -138,7 +140,7 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) {
}
// found the related segment
if seg.StartIP <= s.EndIP && seg.StartIP >= s.StartIP {
if IPCompare(seg.StartIP, s.EndIP) <= 0 && IPCompare(seg.StartIP, s.StartIP) >= 0 {
found = true
}
@ -147,7 +149,7 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) {
}
eList = append(eList, ele)
if seg.EndIP <= s.EndIP {
if IPCompare(seg.EndIP, s.EndIP) <= 0 {
break
}
}
@ -167,10 +169,10 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) {
// segment split
var sList []*Segment
var head = eList[0].Value.(*Segment)
if seg.StartIP > head.StartIP {
if IPCompare(seg.StartIP, head.StartIP) > 0 {
sList = append(sList, &Segment{
StartIP: head.StartIP,
EndIP: seg.StartIP - 1,
EndIP: IPSubOne(seg.StartIP),
Region: head.Region,
})
}
@ -182,9 +184,9 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) {
if len(sList) > 0 {
// check and append the tailing
var tail = eList[len(eList)-1].Value.(*Segment)
if seg.EndIP < tail.EndIP {
if IPCompare(seg.EndIP, tail.EndIP) < 0 {
sList = append(sList, &Segment{
StartIP: seg.EndIP + 1,
StartIP: IPAddOne(seg.EndIP),
EndIP: tail.EndIP,
Region: tail.Region,
})

49
maker/golang/xdb/maker.go
View File

@ -3,7 +3,7 @@
// license that can be found in the LICENSE file.
// ----
// ip2region database v2.0 structure
// Ip2Region database v2.0 structure
//
// +----------------+-------------------+---------------+--------------+
// | header space | speed up index | data payload | block index |
@ -53,20 +53,22 @@ import (
"encoding/binary"
"fmt"
"log/slog"
"math"
"os"
"strings"
"time"
)
const VersionNo = 2
const VersionNo = 3
const HeaderInfoLength = 256
const VectorIndexRows = 256
const VectorIndexCols = 256
const VectorIndexSize = 8
const SegmentIndexSize = 14
const VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize
type Maker struct {
version *Version
srcHandle *os.File
dstHandle *os.File
@ -79,7 +81,7 @@ type Maker struct {
vectorIndex []byte
}
func NewMaker(policy IndexPolicy, srcFile string, dstFile string, fields []int) (*Maker, error) {
func NewMaker(version *Version, policy IndexPolicy, srcFile string, dstFile string, fields []int) (*Maker, error) {
// open the source file with READONLY mode
srcHandle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600)
if err != nil {
@ -93,6 +95,8 @@ func NewMaker(policy IndexPolicy, srcFile string, dstFile string, fields []int)
}
return &Maker{
version: version,
srcHandle: srcHandle,
dstHandle: dstHandle,
@ -117,7 +121,7 @@ func (m *Maker) initDbHeader() error {
// make and write the header space
var header = make([]byte, 256)
// 1, version number
// 1, data version number
binary.LittleEndian.PutUint16(header, uint16(VersionNo))
// 2, index policy code
@ -132,6 +136,9 @@ func (m *Maker) initDbHeader() error {
// 5, index block end ptr
binary.LittleEndian.PutUint32(header[12:], uint32(0))
// 6, ip version
binary.LittleEndian.PutUint16(header[16:], uint16(m.version.Id))
_, err = m.dstHandle.Write(header)
if err != nil {
return err
@ -213,16 +220,16 @@ func (m *Maker) Init() error {
}
// refresh the vector index of the specified ip
func (m *Maker) setVectorIndex(ip uint32, ptr uint32) {
var il0 = (ip >> 24) & 0xFF
var il1 = (ip >> 16) & 0xFF
func (m *Maker) setVectorIndex(ip []byte, ptr uint32) {
var segIdxSize = uint32(m.version.SegmentIndexSize)
var il0, il1 = int(ip[0]), int(ip[1])
var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize
var sPtr = binary.LittleEndian.Uint32(m.vectorIndex[idx:])
if sPtr == 0 {
binary.LittleEndian.PutUint32(m.vectorIndex[idx:], ptr)
binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+SegmentIndexSize)
binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+segIdxSize)
} else {
binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+SegmentIndexSize)
binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+segIdxSize)
}
}
@ -258,6 +265,11 @@ func (m *Maker) Start() error {
return fmt.Errorf("seek to current ptr: %w", err)
}
// @TODO: remove this if the long ptr operation were supported
if pos >= math.MaxUint32 {
return fmt.Errorf("region ptr exceed the max length of %d", math.MaxUint32)
}
_, err = m.dstHandle.Write(region)
if err != nil {
return fmt.Errorf("write region '%s': %w", seg.Region, err)
@ -269,7 +281,7 @@ func (m *Maker) Start() error {
// 2, write the index block and cache the super index block
slog.Info("try to write the segment index block ... ")
var indexBuff = make([]byte, SegmentIndexSize)
var indexBuff = make([]byte, m.version.SegmentIndexSize)
var counter, startIndexPtr, endIndexPtr = 0, int64(-1), int64(-1)
for _, seg := range m.segments {
dataPtr, has := m.regionPool[seg.Region]
@ -285,6 +297,7 @@ func (m *Maker) Start() error {
return fmt.Errorf("empty region info for segment '%s'", seg)
}
var _offset = 0
var segList = seg.Split()
slog.Debug("try to index segment", "length", len(segList), "splits", seg.String())
for _, s := range segList {
@ -293,11 +306,17 @@ func (m *Maker) Start() error {
return fmt.Errorf("seek to segment index block: %w", err)
}
// @TODO: remove this if the long ptr operation were supported
if pos >= math.MaxUint32 {
return fmt.Errorf("segment index ptr exceed the max length of %d", math.MaxUint32)
}
// encode the segment index
binary.LittleEndian.PutUint32(indexBuff, s.StartIP)
binary.LittleEndian.PutUint32(indexBuff[4:], s.EndIP)
binary.LittleEndian.PutUint16(indexBuff[8:], uint16(dataLen))
binary.LittleEndian.PutUint32(indexBuff[10:], dataPtr)
copy(indexBuff[0:], s.StartIP)
copy(indexBuff[len(s.StartIP):], s.EndIP)
_offset = len(s.StartIP) + len(s.EndIP)
binary.LittleEndian.PutUint16(indexBuff[_offset:], uint16(dataLen))
binary.LittleEndian.PutUint32(indexBuff[_offset+2:], dataPtr)
_, err = m.dstHandle.Write(indexBuff)
if err != nil {
return fmt.Errorf("write segment index for '%s': %w", s.String(), err)

36
maker/golang/xdb/searcher.go
View File

@ -3,7 +3,7 @@
// license that can be found in the LICENSE file.
// ---
// ip2region database v2.0 searcher.
// Ip2Region database v2.0 searcher.
// this is part of the maker for testing and validate.
// please use the searcher in binding/golang for production use.
// And this is a Not thread safe implementation.
@ -17,6 +17,8 @@ import (
)
type Searcher struct {
version *Version
handle *os.File
// header info
@ -28,13 +30,15 @@ type Searcher struct {
vectorIndex []byte
}
func NewSearcher(dbFile string) (*Searcher, error) {
func NewSearcher(version *Version, dbFile string) (*Searcher, error) {
handle, err := os.OpenFile(dbFile, os.O_RDONLY, 0600)
if err != nil {
return nil, err
}
return &Searcher{
version: version,
handle: handle,
header: nil,
@ -85,11 +89,10 @@ func (s *Searcher) ClearVectorIndex() {
}
// Search find the region for the specified ip address
func (s *Searcher) Search(ip uint32) (string, int, error) {
func (s *Searcher) Search(ip []byte) (string, int, error) {
// locate the segment index block based on the vector index
var ioCount = 0
var il0 = (ip >> 24) & 0xFF
var il1 = (ip >> 16) & 0xFF
var il0, il1, bytes, tBytes = int(ip[0]), int(ip[1]), len(ip), len(ip) << 1
var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize
var sPtr, ePtr = uint32(0), uint32(0)
if s.vectorIndex != nil {
@ -118,13 +121,14 @@ func (s *Searcher) Search(ip uint32) (string, int, error) {
//log.Printf("vIndex=%s", vIndex)
// binary search the segment index to get the region
var segIndexSize = uint32(s.version.SegmentIndexSize)
var dataLen, dataPtr = 0, uint32(0)
var buff = make([]byte, SegmentIndexSize)
var l, h = 0, int((ePtr - sPtr) / SegmentIndexSize)
var buff = make([]byte, segIndexSize)
var l, h = 0, int((ePtr - sPtr) / segIndexSize)
for l <= h {
// log.Printf("l=%d, h=%d", l, h)
m := (l + h) >> 1
p := sPtr + uint32(m*SegmentIndexSize)
p := sPtr + uint32(m)*segIndexSize
// log.Printf("m=%d, p=%d", m, p)
_, err := s.handle.Seek(int64(p), 0)
if err != nil {
@ -142,18 +146,14 @@ func (s *Searcher) Search(ip uint32) (string, int, error) {
}
// decode the data step by step to reduce the unnecessary calculations
sip := binary.LittleEndian.Uint32(buff)
if ip < sip {
if IPCompare(ip, buff[0:bytes]) < 0 {
h = m - 1
} else if IPCompare(ip, buff[bytes:tBytes]) > 0 {
l = m + 1
} else {
eip := binary.LittleEndian.Uint32(buff[4:])
if ip > eip {
l = m + 1
} else {
dataLen = int(binary.LittleEndian.Uint16(buff[8:]))
dataPtr = binary.LittleEndian.Uint32(buff[10:])
break
}
dataLen = int(binary.LittleEndian.Uint16(buff[tBytes:]))
dataPtr = binary.LittleEndian.Uint32(buff[tBytes+2:])
break
}
}

98
maker/golang/xdb/segment.go
View File

@ -10,8 +10,8 @@ import (
)
type Segment struct {
StartIP uint32
EndIP uint32
StartIP []byte
EndIP []byte
Region string
}
@ -21,17 +21,17 @@ func SegmentFrom(seg string) (*Segment, error) {
return nil, fmt.Errorf("invalid ip segment `%s`", seg)
}
sip, err := CheckIP(ps[0])
sip, err := ParseIP(ps[0])
if err != nil {
return nil, fmt.Errorf("check start ip `%s`: %s", ps[0], err)
}
eip, err := CheckIP(ps[1])
eip, err := ParseIP(ps[1])
if err != nil {
return nil, fmt.Errorf("check end ip `%s`: %s", ps[1], err)
}
if sip > eip {
if IPCompare(sip, eip) > 0 {
return nil, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
}
@ -45,10 +45,10 @@ func SegmentFrom(seg string) (*Segment, error) {
// AfterCheck check the current segment is the one just after the specified one
func (s *Segment) AfterCheck(last *Segment) error {
if last != nil {
if last.EndIP+1 != s.StartIP {
if IPCompare(IPAddOne(last.EndIP), s.StartIP) != 0 {
return fmt.Errorf(
"discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)",
last.EndIP+1, s.StartIP, s.Region,
"discontinuous data segment: last.eip(%s)+1 != seg.sip(%s, %s)",
IP2String(last.EndIP), IP2String(s.StartIP), s.Region,
)
}
}
@ -60,17 +60,39 @@ func (s *Segment) AfterCheck(last *Segment) error {
func (s *Segment) Split() []*Segment {
// 1, split the segment with the first byte
var tList []*Segment
var sByte1, eByte1 = (s.StartIP >> 24) & 0xFF, (s.EndIP >> 24) & 0xFF
var nSip = s.StartIP
var sByte1, eByte1 = int(s.StartIP[0]), int(s.EndIP[0])
// var nSip = s.StartIP
for i := sByte1; i <= eByte1; i++ {
sip := (i << 24) | (nSip & 0xFFFFFF)
eip := (i << 24) | 0xFFFFFF
if eip < s.EndIP {
nSip = (i + 1) << 24
// Make and init the new start & end IP
sip := make([]byte, len(s.StartIP))
eip := make([]byte, len(s.StartIP))
if i == sByte1 {
sip = s.StartIP
} else {
eip = s.EndIP
sip[0] = byte(i)
}
if i == eByte1 {
eip = s.EndIP
} else {
// set the first byte
eip[0] = byte(i)
// fill the buffer with 0xFF
for j := 1; j < len(eip); j++ {
eip[j] = 0xFF
}
}
// sip := (i << 24) | (nSip & 0xFFFFFF)
// eip := (i << 24) | 0xFFFFFF
// if eip < s.EndIP {
// nSip = (i + 1) << 24
// } else {
// eip = s.EndIP
// }
// fmt.Printf("sip:%+v, eip: %+v\n", sip, eip)
// append the new segment (maybe)
tList = append(tList, &Segment{
StartIP: sip,
@ -83,18 +105,41 @@ func (s *Segment) Split() []*Segment {
// 2, split the segments with the second byte
var segList []*Segment
for _, seg := range tList {
base := seg.StartIP & 0xFF000000
nSip := seg.StartIP
sb2, eb2 := (seg.StartIP>>16)&0xFF, (seg.EndIP>>16)&0xFF
// base := seg.StartIP & 0xFF000000
// nSip := seg.StartIP
// sb2, eb2 := (seg.StartIP>>16)&0xFF, (seg.EndIP>>16)&0xFF
sb2, eb2 := int(seg.StartIP[1]), int(seg.EndIP[1])
// fmt.Printf("seg: %s, sb2: %d, eb2: %d\n", seg.String(), sb2, eb2)
for i := sb2; i <= eb2; i++ {
sip := base | (i << 16) | (nSip & 0xFFFF)
eip := base | (i << 16) | 0xFFFF
if eip < seg.EndIP {
nSip = 0
// sip := base | (i << 16) | (nSip & 0xFFFF)
// eip := base | (i << 16) | 0xFFFF
// if eip < seg.EndIP {
// nSip = 0
// } else {
// eip = seg.EndIP
// }
sip := make([]byte, len(s.StartIP))
eip := make([]byte, len(s.StartIP))
sip[0] = seg.StartIP[0]
eip[0] = seg.StartIP[0]
if i == sb2 {
sip = seg.StartIP
} else {
eip = seg.EndIP
sip[1] = byte(i)
}
if i == eb2 {
eip = seg.EndIP
} else {
eip[1] = byte(i)
for j := 2; j < len(eip); j++ {
eip[j] = 0xFF
}
}
// fmt.Printf("i=%d, sip:%+v, eip: %+v\n", i, sip, eip)
segList = append(segList, &Segment{
StartIP: sip,
EndIP: eip,
@ -107,5 +152,10 @@ func (s *Segment) Split() []*Segment {
}
func (s *Segment) String() string {
return fmt.Sprintf("%s|%s|%s", Long2IP(s.StartIP), Long2IP(s.EndIP), s.Region)
return fmt.Sprintf("%s|%s|%s", IP2String(s.StartIP), IP2String(s.EndIP), s.Region)
}
// Contains checks if an IP address is within this segment
func (s *Segment) Contains(ip []byte) bool {
return IPCompare(s.StartIP, ip) <= 0 && IPCompare(ip, s.EndIP) <= 0
}

110
maker/golang/xdb/util.go
View File

@ -7,44 +7,96 @@ package xdb
import (
"bufio"
"fmt"
"math/big"
"net"
"os"
"strconv"
"strings"
)
// Util function
var shiftIndex = []int{24, 16, 8, 0}
func CheckIP(ip string) (uint32, error) {
var ps = strings.Split(ip, ".")
if len(ps) != 4 {
return 0, fmt.Errorf("invalid ip address `%s`", ip)
func ParseIP(ip string) ([]byte, error) {
parsedIP := net.ParseIP(ip)
if parsedIP == nil {
return nil, fmt.Errorf("invalid ip address: %s", ip)
}
var val uint32
for i, s := range ps {
d, err := strconv.Atoi(s)
if err != nil {
return 0, fmt.Errorf("the %dth part `%s` is not an integer", i, s)
}
if d < 0 || d > 255 {
return 0, fmt.Errorf("the %dth part `%s` should be an integer bettween 0 and 255", i, s)
}
val |= uint32(d) << shiftIndex[i]
v4 := parsedIP.To4()
if v4 != nil {
return v4, nil
}
return val, nil
v6 := parsedIP.To16()
if v6 != nil {
return v6, nil
}
return nil, fmt.Errorf("invalid ip address: %s", ip)
}
func Long2IP(ip uint32) string {
return fmt.Sprintf("%d.%d.%d.%d", (ip>>24)&0xFF, (ip>>16)&0xFF, (ip>>8)&0xFF, (ip>>0)&0xFF)
func IP2String(ip []byte) string {
return net.IP(ip[:]).String()
}
func MidIP(sip uint32, eip uint32) uint32 {
return uint32((uint64(sip) + uint64(eip)) >> 1)
func IP2Long(ip []byte) *big.Int {
return big.NewInt(0).SetBytes(ip)
}
// IPCompare compares two IP addresses
// Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2
func IPCompare(ip1, ip2 []byte) int {
for i := 0; i < len(ip1); i++ {
if ip1[i] < ip2[i] {
return -1
}
if ip1[i] > ip2[i] {
return 1
}
}
return 0
}
func IPAddOne(ip []byte) []byte {
var r = make([]byte, len(ip))
copy(r, ip)
for i := len(ip) - 1; i >= 0; i-- {
r[i]++
if r[i] != 0 { // No overflow
break
}
}
return r
}
func IPSubOne(ip []byte) []byte {
var r = make([]byte, len(ip))
copy(r, ip)
for i := len(ip) - 1; i >= 0; i-- {
if r[i] != 0 { // No borrow needed
r[i]--
break
}
r[i] = 0xFF // borrow from the next byte
}
return r
}
func IPMiddle(sip, eip []byte) []byte {
var result = make([]byte, len(sip))
var carry uint16 = 0
// Add the two addresses with carry
for i := len(sip) - 1; i >= 0; i-- {
sum := uint16(sip[i]) + uint16(eip[i]) + carry
result[i] = byte(sum >> 0x01) // Divide by 2
carry = (sum & 0x01) << 7 // Carry for next byte (shift to MSB)
}
return result
}
func IterateSegments(handle *os.File, before func(l string), cb func(seg *Segment) error) error {
@ -70,17 +122,17 @@ func IterateSegments(handle *os.File, before func(l string), cb func(seg *Segmen
return fmt.Errorf("invalid ip segment line `%s`", l)
}
sip, err := CheckIP(ps[0])
sip, err := ParseIP(ps[0])
if err != nil {
return fmt.Errorf("check start ip `%s`: %s", ps[0], err)
}
eip, err := CheckIP(ps[1])
eip, err := ParseIP(ps[1])
if err != nil {
return fmt.Errorf("check end ip `%s`: %s", ps[1], err)
}
if sip > eip {
if IPCompare(sip, eip) > 0 {
return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
}
@ -127,13 +179,13 @@ func CheckSegments(segList []*Segment) error {
var last *Segment
for _, seg := range segList {
// sip must <= eip
if seg.StartIP > seg.EndIP {
if IPCompare(seg.StartIP, seg.EndIP) > 0 {
return fmt.Errorf("segment `%s`: start ip should not be greater than end ip", seg.String())
}
// check the continuity of the data segment
if last != nil {
if last.EndIP+1 != seg.StartIP {
if IPCompare(IPAddOne(last.EndIP), seg.StartIP) != 0 {
return fmt.Errorf("discontinuous segment `%s`: last.eip+1 != cur.sip", seg.String())
}
}

127
maker/golang/xdb/util_test.go
View File

@ -5,44 +5,104 @@
package xdb
import (
"encoding/binary"
"fmt"
"net"
"os"
"testing"
)
func TestCheckIP(t *testing.T) {
var str = "29.34.191.255"
ip, err := CheckIP(str)
if err != nil {
t.Errorf("check ip `%s`: %s\n", str, err)
}
func TestParseIP(t *testing.T) {
var ips = []string{"29.34.191.255", "2c0f:fff0::", "2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}
for _, ip := range ips {
bytes, err := ParseIP(ip)
if err != nil {
t.Errorf("check ip `%s`: %s\n", IP2String(bytes), err)
}
netIP := net.ParseIP(str).To4()
if netIP == nil {
t.Fatalf("parse ip `%s` failed", str)
nip := IP2String(bytes)
fmt.Printf("checkip: (%s / %s), isEqual: %v\n", ip, nip, ip == nip)
}
u32 := binary.BigEndian.Uint32(netIP)
fmt.Printf("checkip: %d, parseip: %d, isEqual: %v\n", ip, u32, ip == u32)
}
func TestLong2IP(t *testing.T) {
var str = "29.34.191.255"
netIP := net.ParseIP(str).To4()
if netIP == nil {
t.Fatalf("parse ip `%s` failed", str)
func TestIPCompare(t *testing.T) {
var ipPairs = [][]string{
{"1.2.3.4", "1.2.3.5"},
{"58.250.36.41", "58.250.30.41"},
{"2c10::", "2e00::"},
{"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"},
{"fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"},
}
u32 := binary.BigEndian.Uint32(netIP)
ipStr := Long2IP(u32)
fmt.Printf("originIP: %s, Long2IP: %s, isEqual: %v\n", str, ipStr, ipStr == str)
for _, pairs := range ipPairs {
fmt.Printf("IPCompare(%s, %s): %d\n", pairs[0], pairs[1], IPCompare([]byte(pairs[0]), []byte(pairs[1])))
}
}
func TestSplitSegment(t *testing.T) {
func TestIPAddOne(t *testing.T) {
var ipPairs = [][]string{
{"1.2.3.4", "1.2.3.5"},
{"2.3.4.5", "2.3.4.6"},
{"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"},
{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"},
{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"},
}
for _, pairs := range ipPairs {
sip, err := ParseIP(pairs[0])
if err != nil {
t.Errorf("parse ip `%s`: %s\n", pairs[0], err)
}
eip, err := ParseIP(pairs[1])
if err != nil {
t.Errorf("parse ip `%s`: %s\n", pairs[1], err)
}
fmt.Printf("IPAddOne(%s) = %s ? %d\n",
pairs[0], pairs[1], IPCompare(IPAddOne(sip), eip))
}
}
func TestIPAddOne2(t *testing.T) {
var ip = []byte{0, 1, 2, 3}
nip := IPAddOne(ip)
fmt.Printf("nip: %+v, ip:%+v", ip, nip)
}
func TestIPSubOne(t *testing.T) {
var ipPairs = [][]string{
{"1.2.3.4", "1.2.3.5"},
{"2.3.4.5", "2.3.4.6"},
{"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"},
{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"},
{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"},
}
for _, pairs := range ipPairs {
sip, err := ParseIP(pairs[0])
if err != nil {
t.Errorf("parse ip `%s`: %s\n", pairs[0], err)
}
eip, err := ParseIP(pairs[1])
if err != nil {
t.Errorf("parse ip `%s`: %s\n", pairs[1], err)
}
fmt.Printf("IPSubOne(%s) = %s ? %d\n",
pairs[1], pairs[0], IPCompare(IPSubOne(eip), sip))
}
}
func TestIPSubOne2(t *testing.T) {
var ip = []byte{0, 1, 2, 3}
nip := IPSubOne(ip)
fmt.Printf("nip: %+v, ip:%+v", ip, nip)
}
func TestSplitSegmentV4(t *testing.T) {
// var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信"
// var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"
// var str = "29.0.0.0|29.34.191.255|美国|0|0|0|0"
var str = "28.201.224.0|29.34.191.255|美国|0|0|0|0"
seg, err := SegmentFrom(str)
if err != nil {
@ -61,8 +121,27 @@ func TestSplitSegment(t *testing.T) {
}
}
func TestSplitSegmentV6(t *testing.T) {
var str = "fec0::|ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff||瑞士|弗里堡州||||专线用户|IANA"
seg, err := SegmentFrom(str)
if err != nil {
t.Fatalf("failed to parser segment '%s': %s", str, err)
}
fmt.Printf("idx: src, seg: %s\n", seg.String())
var segList = seg.Split()
err = CheckSegments(segList)
if err != nil {
t.Fatalf("check segments: %s", err.Error())
}
for i, s := range segList {
fmt.Printf("idx: %3d, seg: %s\n", i, s.String())
}
}
func TestIterateSegments(t *testing.T) {
handle, err := os.OpenFile("../../../data/segments.tests", os.O_RDONLY, 0600)
handle, err := os.OpenFile("../../../data/segments.tests.mixed", os.O_RDONLY, 0600)
if err != nil {
t.Fatalf("failed to open tests file: %s", err)
}

51
maker/golang/xdb/version.go Normal file
View File

@ -0,0 +1,51 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
package xdb
import (
"fmt"
"strings"
)
type Version struct {
Id int
SegmentIndexSize int
}
var (
VX = &Version{}
V4 = &Version{
Id: 4,
SegmentIndexSize: 14, // 4 + 4 + 2 + 4
}
V6 = &Version{
Id: 6,
SegmentIndexSize: 38, // 16 + 16 + 2 + 4
}
)
func VersionFromData(ip string) (*Version, error) {
bytes, err := ParseIP(ip)
if err != nil {
return VX, fmt.Errorf("parse ip fail: %w", err)
}
if len(bytes) == 4 {
return V4, nil
}
return V6, nil
}
func VersionFromName(name string) (*Version, error) {
switch strings.ToUpper(name) {
case "V4", "IPV4":
return V4, nil
case "V6", "IPV6":
return V6, nil
default:
return VX, fmt.Errorf("invalid version name `%s`", name)
}
}