optimize the continuity checking and automatic merging the consecutive segments

maker/golang/main.go

@@ -246,12 +246,11 @@ func testBench() {
 	}
 
 	var count, errCount, tStart = 0, 0, time.Now()
-	var iErr = xdb.IterateSegments(handle, nil, func(sip uint32, eip uint32, region *string) error {
-		var s = *region
-		var l = fmt.Sprintf("%d|%d|%s", sip, eip, s)
+	var iErr = xdb.IterateSegments(handle, nil, func(seg *xdb.Segment) error {
+		var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region)
 		fmt.Printf("try to bench segment: `%s`\n", l)
-		mip := xdb.MidIP(sip, eip)
-		for _, ip := range []uint32{sip, xdb.MidIP(sip, mip), mip, xdb.MidIP(mip, eip), eip} {
+		mip := xdb.MidIP(seg.StartIP, seg.EndIP)
+		for _, ip := range []uint32{seg.StartIP, xdb.MidIP(seg.EndIP, mip), mip, xdb.MidIP(mip, seg.EndIP), seg.EndIP} {
 			fmt.Printf("|-try to bench ip '%s' ... ", xdb.Long2IP(ip))
 			r, _, err := searcher.Search(ip)
 			if err != nil {
@@ -260,9 +259,9 @@ func testBench() {
 
 			// check the region info
 			count++
-			if r != s {
+			if r != seg.Region {
 				errCount++
-				fmt.Printf(" --[Failed] (%s != %s)\n", r, s)
+				fmt.Printf(" --[Failed] (%s != %s)\n", r, seg.Region)
 				if ignoreError == false {
 					return fmt.Errorf("")
 				}

maker/golang/segments.tests

@@ -0,0 +1,5 @@
+192.168.2.1|192.168.2.20|办公室A内网IP
+192.168.2.21|192.168.2.30|办公室A内网IP
+192.168.2.31|192.168.2.60|办公室B内网IP
+192.168.2.61|192.168.2.91|办公室B内网IP
+223.255.236.0|223.255.239.255|中国|0|上海|上海市|电信

maker/golang/xdb/editor.go

@@ -76,19 +76,10 @@ func (e *Editor) loadSegments() error {
 	var last *Segment = nil
 	var tStart = time.Now()
 
-	var err = IterateSegments(e.srcHandle, nil, func(sip uint32, eip uint32, region *string) error {
-		var str = *region
-		var seg = &Segment{
-			StartIP: sip,
-			EndIP:   eip,
-			Region:  str,
-		}
-
+	var err = IterateSegments(e.srcHandle, nil, func(seg *Segment) error {
 		// check the continuity of the data segment
-		if last != nil {
-			if last.EndIP+1 != seg.StartIP {
-				return fmt.Errorf("discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", sip, eip, str)
-			}
+		if err := seg.AfterCheck(last); err != nil {
+			return err
 		}
 
 		last = seg

maker/golang/xdb/maker.go

@@ -138,29 +138,20 @@ func (m *Maker) loadSegments() error {
 	var last *Segment = nil
 	var tStart = time.Now()
 
-	var err = IterateSegments(m.srcHandle, func(l string) {
+	var iErr = IterateSegments(m.srcHandle, func(l string) {
 		log.Printf("load segment: `%s`", l)
-	}, func(sip uint32, eip uint32, region *string) error {
-		var str = *region
-		var seg = &Segment{
-			StartIP: sip,
-			EndIP:   eip,
-			Region:  str,
-		}
-
+	}, func(seg *Segment) error {
 		// check the continuity of the data segment
-		if last != nil {
-			if last.EndIP+1 != seg.StartIP {
-				return fmt.Errorf("discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", sip, eip, str)
-			}
+		if err := seg.AfterCheck(last); err != nil {
+			return err
 		}
 
 		m.segments = append(m.segments, seg)
 		last = seg
 		return nil
 	})
-	if err != nil {
-		return fmt.Errorf("failed to load segments: %s", err)
+	if iErr != nil {
+		return fmt.Errorf("failed to load segments: %s", iErr)
 	}
 
 	log.Printf("all segments loaded, length: %d, elapsed: %s", len(m.segments), time.Since(tStart))

maker/golang/xdb/segment.go

@@ -42,6 +42,20 @@ func SegmentFrom(seg string) (*Segment, error) {
 	}, nil
 }
 
+// AfterCheck check the current segment is the one just after the specified one
+func (s *Segment) AfterCheck(last *Segment) error {
+	if last != nil {
+		if last.EndIP+1 != s.StartIP {
+			return fmt.Errorf(
+				"discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)",
+				last.EndIP+1, s.StartIP, s.Region,
+			)
+		}
+	}
+
+	return nil
+}
+
 // Split the segment based on the pre-two bytes
 func (s *Segment) Split() []*Segment {
 	// 1, split the segment with the first byte

maker/golang/xdb/util.go

@@ -47,7 +47,8 @@ func MidIP(sip uint32, eip uint32) uint32 {
 	return uint32((uint64(sip) + uint64(eip)) >> 1)
 }
 
-func IterateSegments(handle *os.File, before func(l string), cb func(sip uint32, eip uint32, region *string) error) error {
+func IterateSegments(handle *os.File, before func(l string), cb func(seg *Segment) error) error {
+	var last *Segment = nil
 	var scanner = bufio.NewScanner(handle)
 	scanner.Split(bufio.ScanLines)
 	for scanner.Scan() {
@@ -79,9 +80,36 @@ func IterateSegments(handle *os.File, before func(l string), cb func(sip uint32,
 			return fmt.Errorf("empty region info in segment line `%s`", l)
 		}
 
-		if err = cb(sip, eip, &ps[2]); err != nil {
+		var seg = &Segment{
+			StartIP: sip,
+			EndIP:   eip,
+			Region:  ps[2],
+		}
+
+		// check and automatic merging the Consecutive Segments which means:
+		// 1, region info is the same
+		// 2, last.eip+1 = cur.sip
+		if last == nil {
+			last = seg
+			continue
+		} else if last.Region == seg.Region {
+			if err = seg.AfterCheck(last); err == nil {
+				last.EndIP = seg.EndIP
+				continue
+			}
+		}
+
+		if err = cb(last); err != nil {
 			return err
 		}
+
+		// reset the last
+		last = seg
+	}
+
+	// process the last segment
+	if last != nil {
+		return cb(last)
 	}
 
 	return nil

maker/golang/xdb/util_test.go

@@ -8,6 +8,7 @@ import (
 	"encoding/binary"
 	"fmt"
 	"net"
+	"os"
 	"testing"
 )
 
@@ -59,3 +60,17 @@ func TestSplitSegment(t *testing.T) {
 		fmt.Printf("idx: %3d, seg: %s\n", i, s.String())
 	}
 }
+
+func TestIterateSegments(t *testing.T) {
+	handle, err := os.OpenFile("../segments.tests", os.O_RDONLY, 0600)
+	if err != nil {
+		t.Fatalf("failed to open tests file: %s", err)
+	}
+
+	err = IterateSegments(handle, func(l string) {
+		// fmt.Printf("load segment: `%s`\n", l)
+	}, func(seg *Segment) error {
+		fmt.Printf("get segment: `%s`\n", seg)
+		return nil
+	})
+}