diff --git a/.gitignore b/.gitignore index baa7e0e..33056e3 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,4 @@ target #vscode .vscode +build diff --git a/binding/c/Makefile b/binding/c/Makefile index e444283..ea9523b 100644 --- a/binding/c/Makefile +++ b/binding/c/Makefile @@ -6,9 +6,19 @@ xdb_searcher: xdb_searcher.h xdb_searcher.c main.c util_test: xdb_searcher.h xdb_searcher.c util_test.c gcc -g -O2 -I./ xdb_searcher.c util_test.c -o util_test +xdb_searcher.o: xdb_searcher.c + gcc -c -o xdb_searcher.o xdb_searcher.c + +xdb_searcher_lib: xdb_searcher.o + mkdir -p build/lib + mkdir -p build/include + ar -rc build/lib/libxdb_searcher.a `find . -name *.o` + cp xdb_searcher.h build/include + clean: find ./ -name \*.o | xargs rm -f find ./ -name util_test | xargs rm -f find ./ -name xdb_searcher | xargs rm -f + rm -rf build .PHONY: all clean xdb_searcher util_test \ No newline at end of file diff --git a/binding/c/xdb_searcher.h b/binding/c/xdb_searcher.h index 12808bc..8be9429 100644 --- a/binding/c/xdb_searcher.h +++ b/binding/c/xdb_searcher.h @@ -17,7 +17,7 @@ # define XDB_PUBLIC(type) extern __declspec(dllexport) type # define XDB_PRIVATE(type) static type # define XDB_WINDOWS -#elif ( defined(linux) || defined(_UNIX) ) +#elif ( defined(linux) || defined(_UNIX) || defined(__APPLE__) ) # define XDB_PUBLIC(type) extern type # define XDB_PRIVATE(type) static inline type # define XDB_LINUX diff --git a/binding/nginx/README.md b/binding/nginx/README.md index 659cff1..76c51ff 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -8,9 +8,13 @@ $ cd workspace $ wget https://nginx.org/download/nginx-1.23.4.tar.gz $ tar -zxf nginx-1.23.4.tar.gz && rm -rf nginx-1.23.4.tar.gz $ git clone https://github.com/lionsoul2014/ip2region.git -$ cd nginx-1.23.4 -$ ./configure \ - --add-module=$(PWD)/../ip2region/binding/nginx +$ cd ip2region/binding/c +$ make xdb_searcher_lib +$ cd ../../../nginx-1.23.4 +$ ./configure \ + --add-module=$(PWD)/../ip2region/binding/nginx \ + --with-cc-opt="-I $(PWD)/../ip2region/binding/c/build/include" \ + --with-ld-opt="-L$(PWD)/../ip2region/binding/c/build/lib" $ make $ make install ``` @@ -65,4 +69,9 @@ nginx access log sample ``` +./configure \ + --prefix=$(PWD)/../build/ \ + --add-module=$(PWD)/../ip2region/binding/nginx \ + --with-http_realip_module + Made with ♥ by Wu Jian Ping diff --git a/binding/nginx/config b/binding/nginx/config index 08fe212..1da0253 100644 --- a/binding/nginx/config +++ b/binding/nginx/config @@ -1,8 +1,7 @@ ngx_addon_name=ngx_http_ip2region_module NGX_HTTP_IP2REGION_SRCS=" \ - $ngx_addon_dir/src/ngx_http_ip2region_module.c \ - $ngx_addon_dir/src/xdb_searcher.c \ + $ngx_addon_dir/src/ngx_http_ip2region_module.c \ " NGX_HTTP_IP2REGION_DEPS=" \ @@ -13,6 +12,7 @@ if test -n "$ngx_module_link"; then ngx_module_name=$ngx_addon_name ngx_module_deps="$NGX_HTTP_IP2REGION_DEPS" ngx_module_srcs="$NGX_HTTP_IP2REGION_SRCS" + ngx_module_libs="-lxdb_searcher" . auto/module else diff --git a/binding/nginx/src/ngx_http_ip2region_module.h b/binding/nginx/src/ngx_http_ip2region_module.h index 7bfbb7c..8861cee 100644 --- a/binding/nginx/src/ngx_http_ip2region_module.h +++ b/binding/nginx/src/ngx_http_ip2region_module.h @@ -8,7 +8,7 @@ #include #include #include -#include "xdb_searcher.h" +#include typedef struct { xdb_searcher_t searcher; diff --git a/binding/nginx/src/xdb_searcher.c b/binding/nginx/src/xdb_searcher.c deleted file mode 100644 index d763b38..0000000 --- a/binding/nginx/src/xdb_searcher.c +++ /dev/null @@ -1,405 +0,0 @@ -// Copyright 2022 The Ip2Region Authors. All rights reserved. -// Use of this source code is governed by a Apache2.0-style -// license that can be found in the LICENSE file. - -// --- -// @Author Lion -// @Date 2022/06/27 - -// @Modified By Wu Jian Ping -// @Date 2023/03/30 - -#include "sys/time.h" -#include "xdb_searcher.h" - -// internal function prototype define -static int read(xdb_searcher_t *, long offset, char *, size_t length); - -static inline int xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) { - memset(xdb, 0x00, sizeof(xdb_searcher_t)); - - // check the content buffer first - if (c_buffer != NULL) { - xdb->v_index = NULL; - xdb->content = c_buffer; - return 0; - } - - // open the xdb binary file - FILE *handle = fopen(db_path, "rb"); - if (handle == NULL) { - return 1; - } - - xdb->handle = handle; - xdb->v_index = v_index; - - return 0; -} - -// xdb searcher new api define -int xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path) { - return xdb_new_base(xdb, db_path, NULL, NULL); -} - -int xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) { - return xdb_new_base(xdb, db_path, v_index, NULL); -} - -int xdb_new_with_buffer(xdb_searcher_t *xdb, const xdb_content_t *c_buffer) { - return xdb_new_base(xdb, NULL, NULL, c_buffer); -} - -void xdb_close(void *ptr) { - xdb_searcher_t *xdb = (xdb_searcher_t *) ptr; - if (xdb->handle != NULL) { - fclose(xdb->handle); - xdb->handle = NULL; - } -} - -// --- xdb searcher search api define - -int xdb_search_by_string(xdb_searcher_t *xdb, const char *str_ip, char *region_buffer, size_t length) { - unsigned int ip = 0; - int errcode = xdb_check_ip(str_ip, &ip); - if (errcode != 0) { - return 10 + errcode; - } else { - return xdb_search(xdb, ip, region_buffer, length); - } -} - -int xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_buffer, size_t length) { - int il0, il1, idx, err, l, h, m; - unsigned int s_ptr, e_ptr, p, sip, eip, data_ptr; - size_t data_len; - char vector_buffer[xdb_vector_index_size], segment_buffer[xdb_segment_index_size]; - - // reset the io counter - xdb->io_count = 0; - - // locate the segment index block based on the vector index - il0 = ((int) (ip >> 24)) & 0xFF; - il1 = ((int) (ip >> 16)) & 0xFF; - idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size; - if (xdb->v_index != NULL) { - s_ptr = xdb_get_uint(xdb->v_index->buffer, idx); - e_ptr = xdb_get_uint(xdb->v_index->buffer, idx + 4); - } else if (xdb->content != NULL) { - s_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx); - e_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx + 4); - } else { - err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer)); - if (err != 0) { - return 10 + err; - } - - s_ptr = xdb_get_uint(vector_buffer, 0); - e_ptr = xdb_get_uint(vector_buffer, 4); - } - - // printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr); - // binary search to get the final region info - data_len = 0, data_ptr = 0; - l = 0, h = ((int) (e_ptr - s_ptr)) / xdb_segment_index_size; - while (l <= h) { - m = (l + h) >> 1; - p = s_ptr + m * xdb_segment_index_size; - - // read the segment index item - err = read(xdb, p, segment_buffer, sizeof(segment_buffer)); - if (err != 0) { - return 20 + err; - } - - // decode the data fields as needed - sip = xdb_get_uint(segment_buffer, 0); - if (ip < sip) { - h = m - 1; - } else { - eip = xdb_get_uint(segment_buffer, 4); - if (ip > eip) { - l = m + 1; - } else { - data_len = xdb_get_ushort(segment_buffer, 8); - data_ptr = xdb_get_uint(segment_buffer, 10); - break; - } - } - } - - // printf("data_len=%u, data_ptr=%u\n", data_len, data_ptr); - if (data_len == 0) { - region_buffer[0] = '\0'; - return 0; - } - - // buffer length checking - if (data_len >= length) { - return 1; - } - - err = read(xdb, data_ptr, region_buffer, data_len); - if (err != 0) { - return 30 + err; - } - - // auto append a NULL-end - region_buffer[data_len] = '\0'; - return 0; -} - -static inline int read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) { - // check the xdb content cache first - if (xdb->content != NULL) { - memcpy(buffer, xdb->content->buffer + offset, length); - return 0; - } - - // seek to the offset - if (fseek(xdb->handle, offset, SEEK_SET) == -1) { - return 1; - } - - xdb->io_count++; - if (fread(buffer, 1, length, xdb->handle) != length) { - return 2; - } - - return 0; -} - -int xdb_get_io_count(xdb_searcher_t *xdb) { - return xdb->io_count; -} - - -// --- buffer load util functions -xdb_header_t* xdb_load_header(FILE *handle) { - xdb_header_t *header; - unsigned int size = xdb_header_info_length; - - // entry alloc - header = (xdb_header_t *) xdb_malloc(sizeof(xdb_header_t)); - if (header == NULL) { - return NULL; - } - - if (fseek(handle, 0, SEEK_SET) == -1) { - xdb_free(header); - return NULL; - } - - if (fread(header->buffer, 1,size, handle) != size) { - xdb_free(header); - return NULL; - } - - // fill the fields - header->length = size; - header->version = (unsigned short) xdb_get_ushort(header->buffer, 0); - header->index_policy = (unsigned short) xdb_get_ushort(header->buffer, 2); - header->created_at = xdb_get_uint(header->buffer, 4); - header->start_index_ptr = xdb_get_uint(header->buffer, 8); - header->end_index_ptr = xdb_get_uint(header->buffer,12); - - return header; -} - -xdb_header_t* xdb_load_header_from_file(const char *db_path) { - xdb_header_t *header; - FILE *handle = fopen(db_path, "rb"); - if (handle == NULL) { - return NULL; - } - - header = xdb_load_header(handle); - fclose(handle); - return header; -} - -void xdb_close_header(void *ptr) { - xdb_header_t *header = (xdb_header_t *) ptr; - if (header->length > 0) { - header->length = 0; - xdb_free(header); - } -} - -// --- vector index - -xdb_vector_index_t* xdb_load_vector_index(FILE *handle) { - xdb_vector_index_t *v_index; - unsigned int size = xdb_vector_index_length; - - // seek to the vector index offset - if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) { - return NULL; - } - - // do the buffer read - v_index = (xdb_vector_index_t *) xdb_malloc(sizeof(xdb_vector_index_t)); - if (v_index == NULL) { - return NULL; - } - - v_index->length = size; - if (fread(v_index->buffer, 1, size, handle) != size) { - xdb_free(v_index); - return NULL; - } - - return v_index; -} - -xdb_vector_index_t* xdb_load_vector_index_from_file(const char *db_path) { - xdb_vector_index_t *v_index; - FILE *handle = fopen(db_path, "rb"); - if (handle == NULL) { - return NULL; - } - - v_index = xdb_load_vector_index(handle); - fclose(handle); - return v_index; -} - -void xdb_close_vector_index(void *ptr) { - xdb_vector_index_t *v_index = (xdb_vector_index_t *) ptr; - if (v_index->length > 0) { - v_index->length = 0; - xdb_free(v_index); - } -} - -// --- content buffer - -xdb_content_t * xdb_load_content(FILE *handle) { - unsigned int size; - xdb_content_t *content; - - // determine the file size - if (fseek(handle, 0, SEEK_END) == -1) { - return NULL; - } - - size = (unsigned int) ftell(handle); - if (fseek(handle, 0, SEEK_SET) == -1) { - return NULL; - } - - // do the file read - content = (xdb_content_t *) xdb_malloc(sizeof(xdb_content_t)); - if (content == NULL) { - return NULL; - } - - // do the buffer alloc - content->buffer = (char *) xdb_malloc(size); - if (content->buffer == NULL) { - xdb_free(content); - return NULL; - } - - // read the content into the buffer - content->length = size; - if (fread(content->buffer, 1, size, handle) != size) { - xdb_free(content); - return NULL; - } - - return content; -} - -xdb_content_t* xdb_load_content_from_file(const char *db_path) { - xdb_content_t *content; - FILE *handle = fopen(db_path, "rb"); - if (handle == NULL) { - return NULL; - } - - content = xdb_load_content(handle); - fclose(handle); - return content; -} - -void xdb_close_content(void *ptr) { - xdb_content_t *content = (xdb_content_t *) ptr; - if (content->length > 0) { - content->length = 0; - xdb_free(content->buffer); - content->buffer = NULL; - xdb_free(content); - } -} - -// --- End - -// get unsigned long (4bytes) from a specified buffer start from the specified offset -unsigned int xdb_get_uint(const char *buffer, int offset) { - return ( - ((buffer[offset ]) & 0x000000FF) | - ((buffer[offset+1] << 8) & 0x0000FF00) | - ((buffer[offset+2] << 16) & 0x00FF0000) | - ((buffer[offset+3] << 24) & 0xFF000000) - ); -} - -// get unsigned short (2bytes) from a specified buffer start from the specified offset -int xdb_get_ushort(const char *buffer, int offset) { - return ( - ((buffer[offset ]) & 0x000000FF) | - ((buffer[offset+1] << 8) & 0x0000FF00) - ); -} - -// string ip to unsigned int -static int shiftIndex[4] = {24, 16, 8, 0}; -int xdb_check_ip(const char *src_ip, unsigned int *dst_ip) { - char c; - int i, n, ip = 0; - const char *ptr = src_ip; - for (i = 0; i < 4; i++) { - n = 0; - while (1) { - c = *ptr; - ptr++; - if (c >= '0' && c <= '9') { - n *= 10; - n += c - '0'; - } else if ((i < 3 && c == '.') || i == 3) { - // stopping at the '.' but ignore the tailing chars - // after the 3rd one (auto clean the tailing none-integer ?). - break; - } else { - return 1; - } - } - - if (n > 0xFF) { - return 2; - } - - ip |= (n << shiftIndex[i]); - } - - *dst_ip = ip; - return 0; -} - -// unsigned int ip to string ip -void xdb_long2ip(unsigned int ip, char *buffer) { - sprintf(buffer, "%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8) & 0xFF, ip & 0xFF); -} - -// get the middle ip of a and b -unsigned int xdb_mip(unsigned long a, unsigned long b) { - return (unsigned int) ((a + b) >> 1); -} - -long xdb_now() { - struct timeval c_time; - gettimeofday(&c_time, NULL); - return c_time.tv_sec * (int)1e6 + c_time.tv_usec; -} diff --git a/binding/nginx/src/xdb_searcher.h b/binding/nginx/src/xdb_searcher.h deleted file mode 100644 index 4656b49..0000000 --- a/binding/nginx/src/xdb_searcher.h +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2022 The Ip2Region Authors. All rights reserved. -// Use of this source code is governed by a Apache2.0-style -// license that can be found in the LICENSE file. - -// --- -// @Author Lion -// @Date 2022/06/27 - -// @Modified By Wu Jian Ping -// @Date 2023/03/30 - -#ifndef C_XDB_SEARCHER_H -#define C_XDB_SEARCHER_H - -#include -#include -#include - -#define xdb_calloc( _blocks, _bytes ) calloc( _blocks, _bytes ) -#define xdb_malloc( _bytes ) malloc( _bytes ) -#define xdb_free( _ptr ) free( _ptr ) - -// public constants define -#define xdb_header_info_length 256 -#define xdb_vector_index_rows 256 -#define xdb_vector_index_cols 256 -#define xdb_vector_index_size 8 -#define xdb_segment_index_size 14 - -// cache of vector_index_row × vector_index_rows × vector_index_size -#define xdb_vector_index_length 524288 - -typedef struct { - unsigned short version; - unsigned short index_policy; - unsigned int created_at; - unsigned int start_index_ptr; - unsigned int end_index_ptr; - - // the original buffer - unsigned int length; - char buffer[xdb_header_info_length]; -} xdb_header_t; - -xdb_header_t* xdb_load_header(FILE *); - -xdb_header_t *xdb_load_header_from_file(const char *); - -void xdb_close_header(void *); - -// --- vector index buffer -struct xdb_vector_index { - unsigned int length; - char buffer[xdb_vector_index_length]; -}; -typedef struct xdb_vector_index xdb_vector_index_t; - -xdb_vector_index_t* xdb_load_vector_index(FILE *); - -xdb_vector_index_t * xdb_load_vector_index_from_file(const char *); - -void xdb_close_vector_index(void *); - -// --- content buffer -typedef struct { - unsigned int length; - char *buffer; -} xdb_content_t; - -xdb_content_t* xdb_load_content(FILE *); - -xdb_content_t* xdb_load_content_from_file(const char *); - -void xdb_close_content(void *); - -// --- End buffer load - -// xdb searcher structure -typedef struct xdb_searcher_entry { - FILE *handle; - - // header info - const char *header; - int io_count; - - // vector index buffer cache. - // preload the vector index will reduce the number of IO operations - // thus speedup the search process. - const xdb_vector_index_t *v_index; - - // content buffer. - // cache the whole xdb content. - const xdb_content_t *content; -} xdb_searcher_t; - -// xdb searcher new api define -int xdb_new_with_file_only(xdb_searcher_t *, const char *); - -int xdb_new_with_vector_index(xdb_searcher_t *, const char *, const xdb_vector_index_t *); - -int xdb_new_with_buffer(xdb_searcher_t *, const xdb_content_t *); - -void xdb_close(void *); - -// xdb searcher search api define -int xdb_search_by_string(xdb_searcher_t *, const char *, char *, size_t); - -int xdb_search(xdb_searcher_t *, unsigned int, char *, size_t); - -int xdb_get_io_count(xdb_searcher_t *); - -// get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian -unsigned int xdb_get_uint(const char *, int); - -// get unsigned short (2bytes) from a specified buffer start from the specified offset with little-endian -int xdb_get_ushort(const char *, int); - -// check the specified string ip and convert it to an unsigned int -int xdb_check_ip(const char *, unsigned int *); - -// unsigned int ip to string ip -void xdb_long2ip(unsigned int, char *); - -// get the middle ip of a and b -unsigned int xdb_mip(unsigned long, unsigned long); - -// get the current time in microseconds -long xdb_now(); - -#endif //C_XDB_SEARCHER_H