archive-gh-me/tunnel
1
0
Fork 0
mirror of https://github.com/hellobike/tunnel.git synced 2025-12-08 18:02:31 +00:00
Code Issues Projects Releases Wiki Activity

feat:PG同步工具

Browse Source
This commit is contained in:
machunxiao 2018-11-05 16:23:30 +08:00
parent 39e2f57029
commit 5aff34c653
106 changed files with 8491 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
.circleci/config.yml Normal file
View File

@ -0,0 +1,41 @@
# Java Maven CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-java/ for more details
#
version: 2
jobs:
build:
docker:
# specify the version you desire here
- image: circleci/openjdk:8-jdk
# Specify service dependencies here if necessary
# CircleCI maintains a library of pre-built images
# documented at https://circleci.com/docs/2.0/circleci-images/
# - image: circleci/postgres:9.4
working_directory: ~/repo
environment:
# Customize the JVM maximum heap limit
MAVEN_OPTS: -Xmx3200m
steps:
- checkout
# Download and cache dependencies
- restore_cache:
keys:
- v1-dependencies-{{ checksum "pom.xml" }}
# fallback to using the latest cache if no exact match is found
- v1-dependencies-
- run: mvn dependency:go-offline
- save_cache:
paths:
- ~/.m2
key: v1-dependencies-{{ checksum "pom.xml" }}
# run tests!
- run: mvn integration-test

40
.gitignore vendored Normal file
View File

@ -0,0 +1,40 @@
# Compiled class file
*.class
# Log file
*.log
# BlueJ files
*.ctxt
# Mobile Tools for Java (J2ME)
.mtj.tmp/
# Package Files #
*.jar
*.war
*.nar
*.ear
*.zip
*.tar.gz
*.rar
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
# idea
.idea
*.iml
**/target/**
*.ipr
*.iws
out
.DS_Store
# eclipse
.classpath
.project
.settings/
*.imp
logs

288
pom.xml Normal file
View File

@ -0,0 +1,288 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-all</artifactId>
<packaging>pom</packaging>
<version>1.0.0-SNAPSHOT</version>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
<configuration>
<filesets>
<fileset>
<directory>${basedir}/logs</directory>
</fileset>
</filesets>
</configuration>
</plugin>
</plugins>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.1.1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>utf-8</encoding>
</configuration>
<dependencies>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
<version>${asm.version}</version> <!-- Use newer version of ASM -->
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.1.0</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
</plugin>
</plugins>
</pluginManagement>
</build>
<modules>
<module>tunnel-server</module>
<module>tunnel-spi</module>
<module>tunnel-monitor</module>
</modules>
<properties>
<asm.version>6.1.1</asm.version>
<postgresql.version>42.2.5</postgresql.version>
<log4j2.version>2.11.1</log4j2.version>
<disruptor.version>3.4.2</disruptor.version>
<es.version>6.4.2</es.version>
<apollo.version>1.2.0</apollo.version>
<junit.version>4.12</junit.version>
<fastjson.version>1.2.51</fastjson.version>
<commons-lang3.version>3.8.1</commons-lang3.version>
<curator.version>2.12.0</curator.version>
<slf4j.version>1.7.25</slf4j.version>
<jackson.version>2.9.8</jackson.version>
<snappy.version>1.1.7.2</snappy.version>
<druid.version>1.1.12</druid.version>
<guava.version>23.0</guava.version>
<hadoop.version>2.5.1</hadoop.version>
<hbase.version>1.3.0</hbase.version>
<hive.version>2.1.1</hive.version>
<lombok.version>1.18.4</lombok.version>
<tunnel.version>1.0.0-SNAPSHOT</tunnel.version>
<powermock.version>1.7.4</powermock.version>
</properties>
<profiles>
<profile>
<id>dev</id>
<properties>
<env>dev</env>
</properties>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
</profile>
</profiles>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-monitor</artifactId>
<version>${tunnel.version}</version>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-api</artifactId>
<version>${tunnel.version}</version>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-es</artifactId>
<version>${tunnel.version}</version>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-hbase</artifactId>
<version>${tunnel.version}</version>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-hdfs</artifactId>
<version>${tunnel.version}</version>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-hive</artifactId>
<version>${tunnel.version}</version>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-kafka</artifactId>
<version>${tunnel.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>${druid.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.6</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.10</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.11</version>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<version>1.9.3</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.2</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>${disruptor.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>${fastjson.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>

80
tunnel-monitor/pom.xml Normal file
View File

@ -0,0 +1,80 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>tunnel-all</artifactId>
<groupId>com.hellobike.base.tunnel</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>tunnel-monitor</artifactId>
<properties>
<opentracing.version>0.31.0</opentracing.version>
<prometheus.version>0.6.0</prometheus.version>
<opencensus.version>0.18.0</opencensus.version>
<yukon.version>1.0.1</yukon.version>
</properties>
<dependencies>
<dependency>
<groupId>io.opentracing</groupId>
<artifactId>opentracing-api</artifactId>
<version>${opentracing.version}</version>
</dependency>
<!-- The client -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient</artifactId>
<version>${prometheus.version}</version>
</dependency>
<!-- Hotspot JVM metrics-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_hotspot</artifactId>
<version>${prometheus.version}</version>
</dependency>
<!-- Exposition HTTPServer-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_httpserver</artifactId>
<version>${prometheus.version}</version>
</dependency>
<!-- Pushgateway exposition-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_pushgateway</artifactId>
<version>${prometheus.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-api</artifactId>
<version>${opencensus.version}</version>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-impl</artifactId>
<version>${opencensus.version}</version>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-exporter-stats-prometheus</artifactId>
<version>${opencensus.version}</version>
</dependency>
</dependencies>
</project>

43
tunnel-monitor/src/main/java/com/hellobike/base/tunnel/monitor/ExporterConfig.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.monitor;
import lombok.Data;
/**
* @author machunxiao create at 2018-12-27
*/
@Data
public class ExporterConfig {
private int exportPort = 7788;
private String metricName = "TunnelMetric";
private String[] labelName = labelNames();
private String loggerName = "TunnelLogger";
private static String[] labelNames() {
return new String[]{
"slotName", "appId",
"database", "table",
"target", "total",
"currentTime", "error"
};
}
}

59
tunnel-monitor/src/main/java/com/hellobike/base/tunnel/monitor/PrometheusExporter.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.monitor;
import io.opencensus.exporter.stats.prometheus.PrometheusStatsCollector;
import io.prometheus.client.exporter.HTTPServer;
import io.prometheus.client.hotspot.DefaultExports;
/**
* @author machunxiao create at 2018-12-27
*/
public class PrometheusExporter implements TunnelExporter {
private ExporterConfig exporterConfig;
private PrometheusMonitor monitor;
private HTTPServer server;
public PrometheusExporter(ExporterConfig exporterConfig) {
this.exporterConfig = exporterConfig;
this.monitor = new PrometheusMonitor(this.exporterConfig);
}
@Override
public void startup() {
PrometheusStatsCollector.createAndRegister();
DefaultExports.initialize();
try {
this.server = new HTTPServer(this.exporterConfig.getExportPort());
} catch (Exception e) {
//
}
}
@Override
public void destroy() {
if (this.server != null) {
this.server.stop();
}
}
@Override
public TunnelMonitor getTunnelMonitor() {
return monitor;
}
}

39
tunnel-monitor/src/main/java/com/hellobike/base/tunnel/monitor/PrometheusMonitor.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.monitor;
import io.prometheus.client.Gauge;
/**
* @author machunxiao create at 2018-12-25
*/
public class PrometheusMonitor implements TunnelMonitor {
private final Gauge gauge;
public PrometheusMonitor(ExporterConfig config) {
this.gauge = Gauge.build().name(config.getMetricName()).labelNames(config.getMetricName()).help("Tunnel Requests.").register();
}
@Override
public void collect(Statics statics) {
this.gauge.labels(statics.getSlotName(), statics.getAppId(),
statics.getDatabase(), statics.getTable(),
statics.getTarget(), String.valueOf(statics.getTotal()),
String.valueOf(statics.getCurrentTime()), statics.getError()).inc();
}
}

65
tunnel-monitor/src/main/java/com/hellobike/base/tunnel/monitor/Statics.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.monitor;
import lombok.Builder;
import lombok.Data;
import java.util.HashMap;
import java.util.Map;
/**
* @author machunxiao create at 2018-12-13
*/
@Data
@Builder
public class Statics {
private String appId;
private String database;
private String slotName;
private String table;
private int total;
private String target;
private String error;
private long currentTime;
public static Statics createStatics(String appId, String database, String slotName, String table, int total, String target, String error) {
return Statics.builder()
.appId(appId)
.database(database)
.slotName(slotName)
.table(table)
.total(total)
.target(target)
.error(error)
.currentTime(System.currentTimeMillis())
.build();
}
public Map<String, Object> toMap() {
Map<String, Object> map = new HashMap<>(8, 1f);
map.put("appId", getAppId());
map.put("database", getDatabase());
map.put("slotName", getSlotName());
map.put("table", getTable());
map.put("total", getTotal() + "");
map.put("target", getTarget());
map.put("error", getError());
map.put("currentTime", getCurrentTime() == 0 ? System.currentTimeMillis() : getCurrentTime());
return map;
}
}

41
tunnel-monitor/src/main/java/com/hellobike/base/tunnel/monitor/TunnelExporter.java Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.monitor;
/**
* @author machunxiao create at 2018-12-27
*/
public interface TunnelExporter {
/**
* 启动 exporter 容器,以便于外界查询
*/
void startup();
/**
* 关闭 exporter 容器
*/
void destroy();
/**
* 获取监视器
*
* @return 监视器
*/
TunnelMonitor getTunnelMonitor();
}

30
tunnel-monitor/src/main/java/com/hellobike/base/tunnel/monitor/TunnelMonitor.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.monitor;
/**
* @author machunxiao create at 2018-12-25
*/
public interface TunnelMonitor {
/**
* 收集统计数据
*
* @param statics 统计数据
*/
void collect(final Statics statics);
}

50
tunnel-monitor/src/test/java/com/hellobike/base/tunnel/monitor/PrometheusMain.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.monitor;
import java.util.concurrent.CountDownLatch;
import java.util.stream.IntStream;
/**
* @author machunxiao create at 2018-12-27
*/
public class PrometheusMain {
public static void main(String[] args) {
ExporterConfig config = new ExporterConfig();
TunnelExporter exporter = new PrometheusExporter(config);
TunnelMonitor monitor = exporter.getTunnelMonitor();
exporter.startup();
int total = 10;
CountDownLatch latch = new CountDownLatch(total);
Thread[] threads = new Thread[total];
IntStream.range(0, total).forEach(i -> {
threads[i] = new Thread(() -> {
monitor.collect(Statics.createStatics("testApp" + i, "testDb" + i, "sn" + i, "table" + i, i, "target" + i, "error" + i));
latch.countDown();
});
threads[i].start();
});
try {
latch.await();
} catch (Exception e) {
//
}
exporter.destroy();
}
}

223
tunnel-server/pom.xml Normal file
View File

@ -0,0 +1,223 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>tunnel-all</artifactId>
<groupId>com.hellobike.base.tunnel</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>tunnel-server</artifactId>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>utf-8</encoding>
</configuration>
<dependencies>
<dependency>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
<version>${asm.version}</version> <!-- Use newer version of ASM -->
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<mainClass>com.hellobike.base.tunnel.TunnelLauncher</mainClass>
</manifest>
</archive>
<excludes>
<exclude>conf/**</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skipTests>true</skipTests>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<appendAssemblyId>false</appendAssemblyId>
<finalName>AppTunnelService</finalName>
<descriptors>
<descriptor>src/main/assembly/assembly.xml</descriptor>
</descriptors>
<outputDirectory>${project.parent.basedir}/target</outputDirectory>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-monitor</artifactId>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-api</artifactId>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-es</artifactId>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-hbase</artifactId>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-hdfs</artifactId>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-hive</artifactId>
</dependency>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>${postgresql.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>com.ctrip.framework.apollo</groupId>
<artifactId>apollo-client</artifactId>
<version>${apollo.version}</version>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>${curator.version}</version>
<exclusions>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>${curator.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<version>${powermock.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito</artifactId>
<version>${powermock.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>net.manub</groupId>
<artifactId>scalatest-embedded-kafka_2.12</artifactId>
<version>2.0.0</version>
<scope>test</scope>
<exclusions>
<exclusion>
<artifactId>snappy-java</artifactId>
<groupId>org.xerial.snappy</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>pl.allegro.tech</groupId>
<artifactId>embedded-elasticsearch</artifactId>
<version>2.7.1</version>
<scope>test</scope>
<exclusions>
<exclusion>
<artifactId>jackson-core</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-databind</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>

48
tunnel-server/src/main/assembly/assembly.xml Normal file
View File

@ -0,0 +1,48 @@
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3
http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>package</id>
<formats>
<format>zip</format>
</formats>
<fileSets>
<fileSet>
<directory>src/main/resources/configs/${env}/</directory>
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
<directory>src/main/resources/</directory>
<outputDirectory>conf/</outputDirectory>
<excludes>
<exclude>**/configs/**</exclude>
<exclude>**/conf/**</exclude>
</excludes>
</fileSet>
<fileSet>
<directory>bin/${env}</directory>
<outputDirectory>/</outputDirectory>
<fileMode>755</fileMode>
</fileSet>
</fileSets>
<files>
<file>
<source>conf/logrotate</source>
<outputDirectory>/</outputDirectory>
</file>
<file>
<source>bin/check.sh</source>
<outputDirectory>/</outputDirectory>
<fileMode>755</fileMode>
</file>
</files>
<dependencySets>
<dependencySet>
<outputDirectory>/lib</outputDirectory>
<fileMode>0444</fileMode>
</dependencySet>
</dependencySets>
</assembly>

50
tunnel-server/src/main/java/com/hellobike/base/tunnel/TunnelContext.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author machunxiao create at 2018-12-04
*/
public class TunnelContext {
private static final Map<String, TunnelServer> /**/ SERVERS /**/ = new ConcurrentHashMap<>();
private TunnelContext() {
}
public static TunnelServer findServer(String serverId) {
return SERVERS.get(serverId);
}
public static void putServer(TunnelServer server) {
SERVERS.put(server.getServerId(), server);
}
public static void remove(String serverId) {
SERVERS.remove(serverId);
}
public static void close() {
SERVERS.values()
.forEach(TunnelServer::shutdown);
SERVERS.clear();
}
}

379
tunnel-server/src/main/java/com/hellobike/base/tunnel/TunnelLauncher.java Normal file
View File

@ -0,0 +1,379 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel;
import com.alibaba.fastjson.JSON;
import com.hellobike.base.tunnel.apollo.ApolloConfig;
import com.hellobike.base.tunnel.config.*;
import com.hellobike.base.tunnel.filter.TableNameFilter;
import com.hellobike.base.tunnel.monitor.ExporterConfig;
import com.hellobike.base.tunnel.monitor.TunnelExporter;
import com.hellobike.base.tunnel.monitor.TunnelMonitorFactory;
import com.hellobike.base.tunnel.publisher.PublisherManager;
import com.hellobike.base.tunnel.publisher.es.EsPublisher;
import com.hellobike.base.tunnel.publisher.hbase.HBasePublisher;
import com.hellobike.base.tunnel.publisher.kafka.KafkaPublisher;
import com.hellobike.base.tunnel.utils.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.LogManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.lang.management.ManagementFactory;
import java.util.*;
import java.util.stream.Collectors;
/**
* @author machunxiao 2018-11-07
*/
public class TunnelLauncher {
private static final Logger /**/ LOGGER /**/ = LoggerFactory.getLogger(TunnelLauncher.class);
private static final String /**/ TUNNEL_KEY /**/ = "tunnel_subscribe_config";
private static final String /**/ TUNNEL_ZK_KEY /**/ = "tunnel_zookeeper_address";
private static final String /**/ CONFIG_PATH /**/ = "conf/tunnel.properties";
private static final String /**/ APP_ID /**/ = "AppTunnelService";
private static final TunnelConfig /**/ TUNNEL_CONFIG /**/ = new TunnelConfig();
public static void main(String[] args) {
Map<String, String> cmdArgs = toMap(args);
initTunnelConfig(cmdArgs);
initTunnelMonitor(cmdArgs);
ConfigLoader config = ConfigLoaderFactory.getConfigLoader(TUNNEL_CONFIG);
String configValue = config.getProperty(TUNNEL_KEY, "");
if ("".equals(configValue)) {
LOGGER.warn("config is null at first setup");
System.exit(0);
}
LOGGER.info("config value:{}", configValue);
String zkAddress = config.getProperty(TUNNEL_ZK_KEY, "");
if ("".equals(zkAddress)) {
LOGGER.warn("zk address is null");
System.exit(0);
}
ZkConfig zkConfig = new ZkConfig();
zkConfig.setAddress(zkAddress);
LOGGER.info("zk address:{}", zkAddress);
startSubscribe(zkConfig, configValue);
config.addChangeListener(((key, oldValue, newValue) -> {
if (!TUNNEL_KEY.equals(key)) {
return;
}
startSubscribe(zkConfig, newValue);
}));
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
TunnelContext.close();
LogManager.shutdown();
LOGGER.info("TunnelServer Stopped");
}));
LOGGER.info("TunnelServer Started at:{}", TUNNEL_CONFIG.getProcessId());
}
public static TunnelConfig getTunnelConfig() {
return TUNNEL_CONFIG;
}
/**
* 初始化Tunnel 配置
* <pre>
* -d domain
* -a app id
* -u use apollo
* -c config file
* -y use yukon
* </pre>
*
* @param cfg 参数
*/
private static void initTunnelConfig(Map<String, String> cfg) {
TUNNEL_CONFIG.setProcessId(getPid());
TUNNEL_CONFIG.setAppId(cfg.getOrDefault("-a", APP_ID));
TUNNEL_CONFIG.setMetaDomain(cfg.getOrDefault("-d", getMetaDomain()));
TUNNEL_CONFIG.setUseApollo("true".equalsIgnoreCase(cfg.getOrDefault("-u", "true")));
TUNNEL_CONFIG.setUseYukon("true".equalsIgnoreCase(cfg.getOrDefault("-y", "false")));
TUNNEL_CONFIG.setConfigFile(cfg.get("-c"));
}
/**
* <pre>
* -p port
* -m metric name
* -n logger name
* -l labels
* </pre>
*
* @param cfg 参数
*/
private static void initTunnelMonitor(Map<String, String> cfg) {
ExporterConfig config = new ExporterConfig();
String port = cfg.get("-p");
String metricName = cfg.get("-m");
String loggerName = cfg.get("-n");
String labelNames = cfg.get("-l");
if (StringUtils.isNotBlank(port)) {
try {
config.setExportPort(Integer.parseInt(port));
} catch (Exception e) {
}
}
if (StringUtils.isNotBlank(metricName)) {
config.setMetricName(metricName);
}
if (StringUtils.isNotBlank(loggerName)) {
config.setLoggerName(loggerName);
}
if (StringUtils.isNotBlank(labelNames)) {
config.setLabelName(labelNames.split(","));
}
boolean useYukon = "true".equalsIgnoreCase(cfg.getOrDefault("-y", "true"));
TunnelExporter exporter = TunnelMonitorFactory.initializeExporter(useYukon, config);
exporter.startup();
Runtime.getRuntime().addShutdownHook(new Thread(exporter::destroy));
}
private static Map<String, String> toMap(String[] args) {
Map<String, String> cfg = new LinkedHashMap<>();
if (args == null || args.length == 0) {
return cfg;
}
for (int i = 0; i < args.length; i += 2) {
try {
cfg.put(args[i], args[i + 1]);
} catch (Exception e) {
//
}
}
return cfg;
}
private static int getPid() {
String name = ManagementFactory.getRuntimeMXBean().getName();
return Integer.parseInt(name.split("@")[0]);
}
private static void startSubscribe(ZkConfig zkConfig, String value) {
if (StringUtils.isBlank(value)) {
return;
}
ApolloConfig apolloConfig = JSON.parseObject(value, ApolloConfig.class);
List<ApolloConfig.Subscribe> subscribes = apolloConfig.getSubscribes();
for (ApolloConfig.Subscribe subscribe : subscribes) {
TunnelServer newServer = null;
TunnelServer oldServer = null;
try {
SubscribeConfig subscribeConfig = toTunnelConfig(subscribe);
subscribeConfig.setZkConfig(zkConfig);
newServer = new TunnelServer(subscribeConfig);
oldServer = TunnelContext.findServer(newServer.getServerId());
if (oldServer != null) {
oldServer.shutdown();
}
TunnelContext.putServer(newServer);
newServer.start();
} catch (Exception e) {
LOGGER.warn("setup :'" + subscribe.getSlotName() + "' failure", e);
//
if (oldServer != null) {
oldServer.shutdown();
TunnelContext.remove(oldServer.getServerId());
}
if (newServer != null) {
newServer.shutdown();
TunnelContext.remove(newServer.getServerId());
}
}
}
}
private static SubscribeConfig toTunnelConfig(ApolloConfig.Subscribe subscribe) {
String slotName = subscribe.getSlotName();
ApolloConfig.EsConf esConf = subscribe.getEsConf();
ApolloConfig.KafkaConf kafkaConf = subscribe.getKafkaConf();
ApolloConfig.HBaseConf hbaseConf = subscribe.getHbaseConf();
ApolloConfig.HiveConf hiveConf = subscribe.getHiveConf();
ApolloConfig.HdfsConf hdfsConf = subscribe.getHdfsConf();
ApolloConfig.PgConnConf pgConnConf = subscribe.getPgConnConf();
List<ApolloConfig.Rule> rules = subscribe.getRules();
parseEsConfig(slotName, esConf, rules);
parseKafkaConfig(slotName, kafkaConf, rules);
parseHBaseConfig(slotName, hbaseConf, rules);
parseHiveConfig(slotName, hiveConf, rules);
parseHdfsConfig(slotName, hdfsConf, rules);
JdbcConfig jdbcConfig = getJdbcConfig(slotName, pgConnConf);
SubscribeConfig subscribeConfig = new SubscribeConfig();
subscribeConfig.setJdbcConfig(jdbcConfig);
subscribeConfig.setServerId(generateServerId(pgConnConf.getHost(), pgConnConf.getPort(), jdbcConfig.getSlotName()));
return subscribeConfig;
}
/**
* generate a new serverId
*
* @param host host
* @param port port
* @param slot slot
* @return serverId
*/
private static String generateServerId(String host, int port, String slot) {
return slot + "@" + host + ":" + port;
}
private static void parseKafkaConfig(String slotName, ApolloConfig.KafkaConf kafkaConf, List<ApolloConfig.Rule> rules) {
if (kafkaConf == null || kafkaConf.getAddrs() == null || kafkaConf.getAddrs().isEmpty()) {
return;
}
List<KafkaConfig> kafkaConfigs = rules.stream()
.map(TunnelLauncher::toKafkaConfig)
.filter(Objects::nonNull)
.peek(cfg -> cfg.setServer(StringUtils.join(kafkaConf.getAddrs(), ",")))
.collect(Collectors.toList());
PublisherManager.getInstance().putPublisher(slotName, new KafkaPublisher(kafkaConfigs));
}
private static void parseEsConfig(String slotName, ApolloConfig.EsConf esConf, List<ApolloConfig.Rule> rules) {
if (esConf == null || esConf.getAddrs() == null || esConf.getAddrs().isEmpty()) {
return;
}
List<EsConfig> esConfigs = rules.stream()
.map(TunnelLauncher::toEsConfig)
.filter(Objects::nonNull)
.peek(esConfig -> esConfig.setServer(esConf.getAddrs()))
.collect(Collectors.toList());
PublisherManager.getInstance().putPublisher(slotName, new EsPublisher(esConfigs));
}
private static void parseHBaseConfig(String slotName, ApolloConfig.HBaseConf hbaseConf, List<ApolloConfig.Rule> rules) {
if (hbaseConf == null || StringUtils.isBlank(hbaseConf.getZkquorum())) {
return;
}
List<HBaseConfig> configs = rules.stream()
.map(TunnelLauncher::toHBaseConfig)
.filter(Objects::nonNull)
.peek(config -> config.setQuorum(hbaseConf.getZkquorum()))
.collect(Collectors.toList());
PublisherManager.getInstance().putPublisher(slotName, new HBasePublisher(configs));
}
private static void parseHiveConfig(String slotName, ApolloConfig.HiveConf hiveConf, List<ApolloConfig.Rule> rules) {
}
private static void parseHdfsConfig(String slotName, ApolloConfig.HdfsConf hdfsConf, List<ApolloConfig.Rule> rules) {
}
private static KafkaConfig toKafkaConfig(ApolloConfig.Rule rule) {
if (StringUtils.isBlank(rule.getTopic())) {
return null;
}
KafkaConfig kafkaConfig = new KafkaConfig();
kafkaConfig.setTopic(rule.getTopic());
kafkaConfig.setPartition(rule.getPartition());
kafkaConfig.setFilters(Collections.singletonList(new TableNameFilter(rule.getTable())));
kafkaConfig.setPkNames(new ArrayList<>(rule.getPks()));
return kafkaConfig;
}
private static EsConfig toEsConfig(ApolloConfig.Rule rule) {
if (StringUtils.isBlank(rule.getTable())
|| StringUtils.isBlank(rule.getIndex())
|| StringUtils.isBlank(rule.getType())
|| rule.getPks() == null
|| rule.getEsid() == null
) {
return null;
}
EsConfig esConfig = new EsConfig();
esConfig.setTable(rule.getTable());
esConfig.setIndex(rule.getIndex());
esConfig.setType(rule.getType());
esConfig.setPkFieldNames(new ArrayList<>(rule.getPks()));
esConfig.setEsIdFieldNames(new ArrayList<>(rule.getEsid()));
esConfig.setFieldMappings(rule.getFields() == null ? new HashMap<>() : new HashMap<>(rule.getFields()));
esConfig.setFilters(Collections.singletonList(new TableNameFilter(esConfig.getTable())));
esConfig.setSql(rule.getSql());
esConfig.setParameters(rule.getParameters());
return esConfig;
}
private static HBaseConfig toHBaseConfig(ApolloConfig.Rule rule) {
HBaseConfig config = new HBaseConfig();
config.setPks(rule.getPks());
config.setHbaseKey(rule.getHbaseKey());
config.setHbaseTable(rule.getHbaseTable());
config.setTable(rule.getTable());
config.setFamily(StringUtils.isBlank(rule.getFamily()) ? "data" : rule.getFamily());
config.setQualifier(StringUtils.isBlank(rule.getQualifier()) ? "bytes" : rule.getQualifier());
config.setFilters(Collections.singletonList(new TableNameFilter(rule.getTable())));
return config;
}
private static JdbcConfig getJdbcConfig(String slotName, ApolloConfig.PgConnConf pgConnConf) {
String jdbcUrl = "jdbc:postgresql://" + pgConnConf.getHost() + ":" + pgConnConf.getPort() + "/" + pgConnConf.getDatabase();
JdbcConfig jdbcConfig = new JdbcConfig();
jdbcConfig.setSlotName(slotName);
jdbcConfig.setUrl(jdbcUrl);
jdbcConfig.setUsername(pgConnConf.getUser());
jdbcConfig.setPassword(pgConnConf.getPassword());
jdbcConfig.setHost(pgConnConf.getHost());
jdbcConfig.setPort(pgConnConf.getPort());
jdbcConfig.setSchema(pgConnConf.getDatabase());
return jdbcConfig;
}
private static String getMetaDomain() {
InputStream is = FileUtils.load(CONFIG_PATH);
if (is == null) {
return "";
}
Properties prop = new Properties();
try {
prop.load(is);
} catch (IOException e) {
//
}
return prop.getProperty("tunnel.apollo.meta.domain", "");
}
}

246
tunnel-server/src/main/java/com/hellobike/base/tunnel/TunnelServer.java Normal file
View File

@ -0,0 +1,246 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel;
import com.hellobike.base.tunnel.config.JdbcConfig;
import com.hellobike.base.tunnel.config.SubscribeConfig;
import com.hellobike.base.tunnel.ha.ZkLock;
import com.hellobike.base.tunnel.model.InvokeContext;
import com.hellobike.base.tunnel.parse.EventParser;
import com.hellobike.base.tunnel.parse.IEventParser;
import com.hellobike.base.tunnel.utils.TimeUtils;
import org.postgresql.PGConnection;
import org.postgresql.PGProperty;
import org.postgresql.replication.LogSequenceNumber;
import org.postgresql.replication.PGReplicationStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
/**
* @author machunxiao create at 2018-11-23
*/
public class TunnelServer {
private static final Logger /**/ log = LoggerFactory.getLogger(TunnelServer.class);
private final String /**/ serverId;
private final SubscribeConfig /**/ config;
private final JdbcConfig /**/ jdbcConfig;
private final IEventParser /**/ eventParser = new EventParser();
private final ZkLock /**/ zkLock;
private final String /**/ slotName;
private final ReentrantLock /**/ lock = new ReentrantLock();
private final Thread /**/ startThread;
private final Thread /**/ receiveThread;
private volatile boolean /**/ started = false;
private Connection /**/ connection;
private PGConnection /**/ rplConnection;
private PGReplicationStream /**/ stream;
public TunnelServer(SubscribeConfig config) {
this.serverId = config.getServerId();
this.config = config;
this.jdbcConfig = config.getJdbcConfig();
this.slotName = this.jdbcConfig.getSlotName();
this.zkLock = new ZkLock(this.config.getZkConfig().getAddress(), generateLockKey(config.getJdbcConfig()));
this.startThread = new Thread(new StartTask(), "TunnelStartThread-" + this.slotName);
this.receiveThread = new Thread(new ReceiveTask(), "TunnelReceiveThread-" + this.slotName);
}
private static String generateLockKey(JdbcConfig config) {
return "/com/hellobike/base/tunnel/lock/" + config.getHost() + ":" + config.getPort() + "/" + config.getSchema() + "/" + config.getSlotName();
}
private static void closeClosable(AutoCloseable closeable) {
if (closeable != null) {
try {
closeable.close();
} catch (Exception e) {
//
}
}
}
public void start() {
this.startThread.start();
}
public void shutdown() {
started = false;
closeClosable(this.stream);
closeClosable(this.connection);
zkLock.unlock();
zkLock.close();
}
public String getServerId() {
return serverId;
}
private void createRplConn() throws SQLException {
String url = this.jdbcConfig.getUrl();
Properties props = new Properties();
PGProperty.USER.set(props, this.jdbcConfig.getUsername());
PGProperty.PASSWORD.set(props, this.jdbcConfig.getPassword());
PGProperty.ASSUME_MIN_SERVER_VERSION.set(props, this.jdbcConfig.getMinVersion());
PGProperty.REPLICATION.set(props, this.jdbcConfig.getRplLevel());
PGProperty.PREFER_QUERY_MODE.set(props, "simple");
this.connection = DriverManager.getConnection(url, props);
this.rplConnection = this.connection.unwrap(PGConnection.class);
log.info("GetRplConnection success,slot:{}", this.slotName);
}
private void createRplSlot() throws SQLException {
try {
this.rplConnection.getReplicationAPI()
.createReplicationSlot()
.logical()
.withSlotName(this.jdbcConfig.getSlotName())
.withOutputPlugin("test_decoding")
.make();
} catch (SQLException e) {
String msg = "ERROR: replication slot \"" + this.jdbcConfig.getSlotName() + "\" already exists";
if (msg.equals(e.getMessage())) {
return;
}
throw e;
}
log.info("GetRplSlot success,slot:{}", this.slotName);
}
private void createRplStream() throws SQLException {
this.stream = this.rplConnection.getReplicationAPI()
.replicationStream()
.logical()
.withSlotName(this.jdbcConfig.getSlotName())
.withSlotOption("include-xids", true)
.withSlotOption("skip-empty-xacts", true)
.withStatusInterval(5, TimeUnit.SECONDS)
.start();
log.info("GetRplStream success,slot:{}", this.slotName);
}
private void receiveStream() throws SQLException {
assert !stream.isClosed();
assert !connection.isClosed();
//non blocking receive message
ByteBuffer msg = stream.readPending();
if (msg == null) {
TimeUtils.sleepInMills(10L);
return;
}
int offset = msg.arrayOffset();
byte[] source = msg.array();
int length = source.length - offset;
LogSequenceNumber lsn = stream.getLastReceiveLSN();
InvokeContext ctx = new InvokeContext();
ctx.setMessage(new String(source, offset, length));
ctx.setJdbcUrl(this.jdbcConfig.getUrl());
ctx.setJdbcUser(this.jdbcConfig.getUsername());
ctx.setJdbcPass(this.jdbcConfig.getPassword());
ctx.setSlotName(this.slotName);
ctx.setServerId(this.serverId);
ctx.setLsn(lsn.asLong());
eventParser.parse(ctx);
//feedback
stream.setAppliedLSN(lsn);
stream.setFlushedLSN(lsn);
}
private void recover() {
this.lock.lock();
try {
long s = System.currentTimeMillis();
closeClosable(stream);
closeClosable(connection);
while (true) {
try {
createRplConn();
createRplSlot();
createRplStream();
break;
} catch (Exception e) {
log.warn("Recover Streaming Occurred Error", e);
closeClosable(stream);
closeClosable(connection);
TimeUtils.sleepInMills(5000);
}
}
long e = System.currentTimeMillis();
log.info("recover logical replication success,slot:{},cost:{}ms", slotName, e - s);
} finally {
this.lock.unlock();
}
}
private class StartTask implements Runnable {
@Override
public void run() {
if (zkLock.tryLock()) {
try {
createRplConn();
createRplSlot();
createRplStream();
started = true;
receiveThread.start();
log.warn("Startup RplStream Success");
} catch (Exception e) {
log.warn("Startup RplStream Failure", e);
shutdown();
}
}
}
}
private class ReceiveTask implements Runnable {
@Override
public void run() {
while (started) {
try {
receiveStream();
} catch (Exception e) {
log.warn("receive msg failure,try to recover.", e);
recover();
TimeUtils.sleepInMills(2000);
}
}
}
}
}

418
tunnel-server/src/main/java/com/hellobike/base/tunnel/apollo/ApolloConfig.java Normal file
View File

@ -0,0 +1,418 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.apollo;
import com.alibaba.fastjson.annotation.JSONField;
import java.util.List;
import java.util.Map;
/**
* 这个是兼容代码
*
* @author machunxiao 2018-11-01
*/
public class ApolloConfig {
@JSONField(name = "pg_dump_path")
private String pgDumpPath;
private List<Subscribe> subscribes;
public String getPgDumpPath() {
return pgDumpPath;
}
public void setPgDumpPath(String pgDumpPath) {
this.pgDumpPath = pgDumpPath;
}
public List<Subscribe> getSubscribes() {
return subscribes;
}
public void setSubscribes(List<Subscribe> subscribes) {
this.subscribes = subscribes;
}
public static class Subscribe {
private String slotName;
private PgConnConf pgConnConf;
private List<Rule> rules;
private KafkaConf kafkaConf;
private EsConf esConf;
private HBaseConf hbaseConf;
private HiveConf hiveConf;
private HdfsConf hdfsConf;
public String getSlotName() {
return slotName;
}
public void setSlotName(String slotName) {
this.slotName = slotName;
}
public PgConnConf getPgConnConf() {
return pgConnConf;
}
public void setPgConnConf(PgConnConf pgConnConf) {
this.pgConnConf = pgConnConf;
}
public List<Rule> getRules() {
return rules;
}
public void setRules(List<Rule> rules) {
this.rules = rules;
}
public KafkaConf getKafkaConf() {
return kafkaConf;
}
public void setKafkaConf(KafkaConf kafkaConf) {
this.kafkaConf = kafkaConf;
}
public EsConf getEsConf() {
return esConf;
}
public void setEsConf(EsConf esConf) {
this.esConf = esConf;
}
public HBaseConf getHbaseConf() {
return hbaseConf;
}
public void setHbaseConf(HBaseConf hbaseConf) {
this.hbaseConf = hbaseConf;
}
public HiveConf getHiveConf() {
return hiveConf;
}
public void setHiveConf(HiveConf hiveConf) {
this.hiveConf = hiveConf;
}
public HdfsConf getHdfsConf() {
return hdfsConf;
}
public void setHdfsConf(HdfsConf hdfsConf) {
this.hdfsConf = hdfsConf;
}
}
public static class PgConnConf {
private String host;
private int port;
private String database;
private String schema;
private String user;
private String password;
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public int getPort() {
return port;
}
public void setPort(int port) {
this.port = port;
}
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getSchema() {
return schema;
}
public void setSchema(String schema) {
this.schema = schema;
}
public String getUser() {
return user;
}
public void setUser(String user) {
this.user = user;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
}
public static class KafkaConf {
private List<String> addrs;
public List<String> getAddrs() {
return addrs;
}
public void setAddrs(List<String> addrs) {
this.addrs = addrs;
}
}
public static class EsConf {
private String addrs;
public String getAddrs() {
return addrs;
}
public void setAddrs(String addrs) {
this.addrs = addrs;
}
}
public static class HBaseConf {
private String zkquorum;
public String getZkquorum() {
return zkquorum;
}
public void setZkquorum(String zkquorum) {
this.zkquorum = zkquorum;
}
}
public static class HiveConf {
}
public static class HdfsConf {
}
public static class Rule {
private String table;
private String topic;
private int partition;
private List<String> pks;
private List<String> esid;
private String index;
private String type;
private Map<String, String> fields;
private String sql;
private List<String> parameters;
private String family;
private String qualifier;
private List<String> rowKeys;
private String hbaseTable;
private List<String> hbaseKey;
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public int getPartition() {
return partition;
}
public void setPartition(int partition) {
this.partition = partition;
}
public List<String> getPks() {
return pks;
}
public void setPks(List<String> pks) {
this.pks = pks;
}
public List<String> getEsid() {
return esid;
}
public void setEsid(List<String> esid) {
this.esid = esid;
}
public String getIndex() {
return index;
}
public void setIndex(String index) {
this.index = index;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public Map<String, String> getFields() {
return fields;
}
public void setFields(Map<String, String> fields) {
this.fields = fields;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
public List<String> getParameters() {
return parameters;
}
public void setParameters(List<String> parameters) {
this.parameters = parameters;
}
public String getFamily() {
return family;
}
public void setFamily(String family) {
this.family = family;
}
public String getQualifier() {
return qualifier;
}
public void setQualifier(String qualifier) {
this.qualifier = qualifier;
}
public List<String> getRowKeys() {
return rowKeys;
}
public void setRowKeys(List<String> rowKeys) {
this.rowKeys = rowKeys;
}
public String getHbaseTable() {
return hbaseTable;
}
public void setHbaseTable(String hbaseTable) {
this.hbaseTable = hbaseTable;
}
public List<String> getHbaseKey() {
return hbaseKey;
}
public void setHbaseKey(List<String> hbaseKey) {
this.hbaseKey = hbaseKey;
}
}
/**
* <pre>
* 例如在es索引中,一个复合索引 es_idx_t1包含10个字段,其中5个来自tb1,5个来自tb2,
* 当tb1或者tb2有新增或删除的时候,查询另外一个表的内容,添加到索引中
*
* val: insert into tb1(id1,name1,age1) values('id1','name1','age1');
*
* sql: select id2,name2,age2 from tb2 where id = @id and name= @name
*
* idx: append into es1(id1,id2,name1,name2,age1,age2) values()
*
* 假设目前只支持根据主键的join
*
* </pre>
*/
public static class Join {
private String table;
private String sql;
private List<String> parameters;
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
public List<String> getParameters() {
return parameters;
}
public void setParameters(List<String> parameters) {
this.parameters = parameters;
}
}
}

73
tunnel-server/src/main/java/com/hellobike/base/tunnel/apollo/ApolloConfigLoader.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.apollo;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigChangeListener;
import com.ctrip.framework.apollo.ConfigService;
import com.ctrip.framework.apollo.model.ConfigChange;
import com.ctrip.framework.apollo.model.ConfigChangeEvent;
import com.hellobike.base.tunnel.config.ConfigListener;
import com.hellobike.base.tunnel.config.ConfigLoader;
import java.util.Set;
/**
* @author machunxiao create at 2018-12-26
*/
public class ApolloConfigLoader implements ConfigLoader {
private final Config config;
public ApolloConfigLoader(String appId, String metaDomain) {
this.config = ConfigService.getAppConfig();
}
@Override
public String getProperty(String key, String defaultValue) {
return this.config.getProperty(key, defaultValue);
}
@Override
public void addChangeListener(ConfigListener configListener) {
this.config.addChangeListener(new ApolloInnerConfigListener(configListener));
}
private static class ApolloInnerConfigListener implements ConfigChangeListener {
private final ConfigListener configListener;
public ApolloInnerConfigListener(ConfigListener configListener) {
this.configListener = configListener;
}
@Override
public void onChange(ConfigChangeEvent changeEvent) {
Set<String> keys = changeEvent.changedKeys();
for (String key : keys) {
ConfigChange change = changeEvent.getChange(key);
if (change == null) {
continue;
}
String oldValue = change.getOldValue();
String newValue = change.getNewValue();
configListener.onChange(key, oldValue, newValue);
}
}
}
}

34
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/ConfigListener.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.config;
/**
* @author machunxiao create at 2018-12-26
*/
public interface ConfigListener {
/**
* 监听值变化
*
* @param key 键值
* @param oldValue 旧值
* @param newValue 新值
*/
void onChange(String key, String oldValue, String newValue);
}

40
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/ConfigLoader.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.config;
/**
* @author machunxiao create at 2018-12-26
*/
public interface ConfigLoader {
/**
* 获取配置
*
* @param key key
* @param defaultValue 默认值
* @return
*/
String getProperty(String key, String defaultValue);
/**
* 添加事件监听器
*
* @param configListener 监听器
*/
void addChangeListener(ConfigListener configListener);
}

36
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/ConfigLoaderFactory.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.config;
import com.hellobike.base.tunnel.apollo.ApolloConfigLoader;
import com.hellobike.base.tunnel.config.file.FileConfigLoader;
/**
* @author machunxiao create at 2018-12-27
*/
public class ConfigLoaderFactory {
private ConfigLoaderFactory() {
}
public static ConfigLoader getConfigLoader(TunnelConfig tunnelConfig) {
if (tunnelConfig.isUseApollo()) {
return new ApolloConfigLoader(tunnelConfig.getAppId(), tunnelConfig.getMetaDomain());
}
return new FileConfigLoader(tunnelConfig.getConfigFile());
}
}

151
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/EsConfig.java Normal file
View File

@ -0,0 +1,151 @@
package com.hellobike.base.tunnel.config;
import com.hellobike.base.tunnel.filter.IEventFilter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-11-01
*/
public class EsConfig {
private String server;
private String schema;
private String table;
private String index;
private String type;
private Map<String, String> fieldMappings;
private List<String> pkFieldNames;
private List<String> esIdFieldNames;
private String separator = ";";
private String dstTable;
private String sql;
private List<String> parameters;
private List<IEventFilter> filters = new ArrayList<>();
public String getServer() {
return server;
}
public void setServer(String server) {
this.server = server;
}
public String getSchema() {
return schema;
}
public void setSchema(String schema) {
this.schema = schema;
}
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public String getIndex() {
return index;
}
public void setIndex(String index) {
this.index = index;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public Map<String, String> getFieldMappings() {
return fieldMappings;
}
public void setFieldMappings(Map<String, String> fieldMappings) {
this.fieldMappings = fieldMappings;
}
public List<String> getPkFieldNames() {
return pkFieldNames;
}
public void setPkFieldNames(List<String> pkFieldNames) {
this.pkFieldNames = pkFieldNames;
}
public List<String> getEsIdFieldNames() {
return esIdFieldNames;
}
public void setEsIdFieldNames(List<String> esIdFieldNames) {
this.esIdFieldNames = esIdFieldNames;
}
public String getSeparator() {
return separator;
}
public void setSeparator(String separator) {
this.separator = separator;
}
public List<IEventFilter> getFilters() {
return filters;
}
public void setFilters(List<IEventFilter> filters) {
this.filters = filters;
}
public String getDstTable() {
return dstTable;
}
public void setDstTable(String dstTable) {
this.dstTable = dstTable;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
public List<String> getParameters() {
return parameters;
}
public void setParameters(List<String> parameters) {
this.parameters = parameters;
}
}

103
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/HBaseConfig.java Normal file
View File

@ -0,0 +1,103 @@
package com.hellobike.base.tunnel.config;
import com.hellobike.base.tunnel.filter.IEventFilter;
import java.util.List;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-10
*/
public class HBaseConfig {
private String table;
private List<String> pks;
private String hbaseTable;
private List<String> hbaseKey;
private String family;
private String qualifier;
private String quorum;
private List<IEventFilter> filters;
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public List<String> getPks() {
return pks;
}
public void setPks(List<String> pks) {
this.pks = pks;
}
public String getHbaseTable() {
return hbaseTable;
}
public void setHbaseTable(String hbaseTable) {
this.hbaseTable = hbaseTable;
}
public List<String> getHbaseKey() {
return hbaseKey;
}
public void setHbaseKey(List<String> hbaseKey) {
this.hbaseKey = hbaseKey;
}
public String getFamily() {
return family;
}
public void setFamily(String family) {
this.family = family;
}
public String getQualifier() {
return qualifier;
}
public void setQualifier(String qualifier) {
this.qualifier = qualifier;
}
public String getQuorum() {
return quorum;
}
public void setQuorum(String quorum) {
this.quorum = quorum;
}
public List<IEventFilter> getFilters() {
return filters;
}
public void setFilters(List<IEventFilter> filters) {
this.filters = filters;
}
}

23
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/HdfsConfig.java Normal file
View File

@ -0,0 +1,23 @@
package com.hellobike.base.tunnel.config;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-10
*/
public class HdfsConfig {
}

23
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/HiveConfig.java Normal file
View File

@ -0,0 +1,23 @@
package com.hellobike.base.tunnel.config;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-10
*/
public class HiveConfig {
}

113
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/JdbcConfig.java Normal file
View File

@ -0,0 +1,113 @@
package com.hellobike.base.tunnel.config;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-11-01
*/
public class JdbcConfig {
private String url;
private String username;
private String password;
private String slotName;
private String lastLsn = "";
private String minVersion = "9.4";
private String rplLevel = "database";
private String host;
private int port;
private String schema;
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getSlotName() {
return slotName;
}
public void setSlotName(String slotName) {
this.slotName = slotName;
}
public String getLastLsn() {
return lastLsn;
}
public void setLastLsn(String lastLsn) {
this.lastLsn = lastLsn;
}
public String getMinVersion() {
return minVersion;
}
public void setMinVersion(String minVersion) {
this.minVersion = minVersion;
}
public String getRplLevel() {
return rplLevel;
}
public void setRplLevel(String rplLevel) {
this.rplLevel = rplLevel;
}
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public int getPort() {
return port;
}
public void setPort(int port) {
this.port = port;
}
public String getSchema() {
return schema;
}
public void setSchema(String schema) {
this.schema = schema;
}
}

110
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/KafkaConfig.java Normal file
View File

@ -0,0 +1,110 @@
package com.hellobike.base.tunnel.config;
import com.hellobike.base.tunnel.filter.IEventFilter;
import java.util.List;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-11-01
*/
public class KafkaConfig {
private String topic;
private String server;
private String ackConfig = "all";
private int retryTimes;
private String keySerializer = "org.apache.kafka.common.serialization.StringSerializer";
private String valSerializer = "org.apache.kafka.common.serialization.StringSerializer";
private int partition = 0;
private List<IEventFilter> filters;
private List<String> pkNames;
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public String getServer() {
return server;
}
public void setServer(String server) {
this.server = server;
}
public String getAckConfig() {
return ackConfig;
}
public void setAckConfig(String ackConfig) {
this.ackConfig = ackConfig;
}
public int getRetryTimes() {
return retryTimes;
}
public void setRetryTimes(int retryTimes) {
this.retryTimes = retryTimes;
}
public String getKeySerializer() {
return keySerializer;
}
public void setKeySerializer(String keySerializer) {
this.keySerializer = keySerializer;
}
public String getValSerializer() {
return valSerializer;
}
public void setValSerializer(String valSerializer) {
this.valSerializer = valSerializer;
}
public int getPartition() {
return partition;
}
public void setPartition(int partition) {
this.partition = partition;
}
public List<IEventFilter> getFilters() {
return filters;
}
public void setFilters(List<IEventFilter> filters) {
this.filters = filters;
}
public List<String> getPkNames() {
return pkNames;
}
public void setPkNames(List<String> pkNames) {
this.pkNames = pkNames;
}
}

53
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/SubscribeConfig.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.config;
/**
* @author machunxiao 2018-10-26
*/
public class SubscribeConfig {
private String serverId;
private JdbcConfig jdbcConfig;
private ZkConfig zkConfig;
public String getServerId() {
return serverId;
}
public void setServerId(String serverId) {
this.serverId = serverId;
}
public JdbcConfig getJdbcConfig() {
return jdbcConfig;
}
public void setJdbcConfig(JdbcConfig jdbcConfig) {
this.jdbcConfig = jdbcConfig;
}
public ZkConfig getZkConfig() {
return zkConfig;
}
public void setZkConfig(ZkConfig zkConfig) {
this.zkConfig = zkConfig;
}
}

35
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/TunnelConfig.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.config;
import lombok.Data;
/**
* @author machunxiao create at 2018-12-27
*/
@Data
public class TunnelConfig {
private String appId;
private String metaDomain;
private String configFile;
private boolean useApollo;
private boolean useYukon;
private int processId;
}

34
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/ZkConfig.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.config;
/**
* @author machunxiao 2018-11-07
*/
public class ZkConfig {
private String address = "localhost:2181";
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
}

199
tunnel-server/src/main/java/com/hellobike/base/tunnel/config/file/FileConfigLoader.java Normal file
View File

@ -0,0 +1,199 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.config.file;
import com.hellobike.base.tunnel.config.ConfigListener;
import com.hellobike.base.tunnel.config.ConfigLoader;
import com.hellobike.base.tunnel.utils.NamedThreadFactory;
import java.io.File;
import java.io.FileInputStream;
import java.nio.file.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* @author machunxiao create at 2018-12-26
*/
public class FileConfigLoader implements ConfigLoader, AutoCloseable {
private File file;
private Map<String, String> properties = new ConcurrentHashMap<>();
private List<ConfigListener> listeners = new CopyOnWriteArrayList<>();
private ExecutorService executor = new ThreadPoolExecutor(1, 1, 60, TimeUnit.SECONDS, new ArrayBlockingQueue<>(1), new NamedThreadFactory("FileWatcherThread"));
private AtomicBoolean started = new AtomicBoolean(Boolean.FALSE);
public FileConfigLoader(String fileName) {
this.file = new File(fileName);
this.properties.putAll(load(this.file));
addFileWatcher(this.file);
}
private static Map<String, String> load(File file) {
Map<String, String> data = new LinkedHashMap<>();
try {
Properties prop = new Properties();
prop.load(new FileInputStream(file));
for (Map.Entry<Object, Object> e : prop.entrySet()) {
Object key = e.getKey();
Object value = e.getValue();
if (key != null) {
data.put((String) key, (String) value);
}
}
} catch (Exception e) {
}
return data;
}
@Override
public String getProperty(String key, String defaultValue) {
return properties.getOrDefault(key, defaultValue);
}
@Override
public void addChangeListener(ConfigListener configListener) {
if (!this.listeners.contains(configListener)) {
this.listeners.add(configListener);
}
}
@Override
public void close() {
this.started.compareAndSet(Boolean.TRUE, Boolean.FALSE);
this.executor.shutdown();
}
private void addFileWatcher(File file) {
FileWatchTask fileWatchTask = new FileWatchTask(file);
this.started.compareAndSet(Boolean.FALSE, Boolean.TRUE);
this.executor.submit(fileWatchTask);
}
private static class ThreeTuple<O1, O2, O3> {
private O1 k1;
private O2 v1;
private O3 v2;
private ThreeTuple(O1 k1, O2 v1, O3 v2) {
this.k1 = k1;
this.v1 = v1;
this.v2 = v2;
}
private boolean valueEquals() {
if (v1 == null) {
return v2 == null;
}
return v1.equals(v2);
}
}
private class FileWatchTask implements Runnable {
private File file;
private FileWatchTask(File file) {
this.file = file;
}
@Override
public void run() {
try (WatchService watchService = FileSystems.getDefault().newWatchService()) {
Path dir = Paths.get(this.file.getParent());
dir.register(watchService, StandardWatchEventKinds.ENTRY_MODIFY);
while (started.get()) {
try {
WatchKey key = watchService.poll(50, TimeUnit.MILLISECONDS);
if (key == null) {
continue;
}
for (WatchEvent<?> event : key.pollEvents()) {
WatchEvent.Kind<?> kind = event.kind();
List<ThreeTuple<String, String, String>> data = new ArrayList<>();
Path path = (Path) event.context();
if (!path.toFile().getName().equals(this.file.getName())) {
continue;
}
if (kind == StandardWatchEventKinds.ENTRY_MODIFY) {
Map<String, String> newData = load(this.file);
mergeData(properties, newData, data);
}
onDataChange(data);
}
key.reset();
} catch (Throwable e) {
//
}
}
} catch (Throwable t) {
//
}
}
private synchronized void mergeData(Map<String, String> oldProp, Map<String, String> newProp, List<ThreeTuple<String, String, String>> data) {
Map<String, ThreeTuple<String, String, String>> tmp = new LinkedHashMap<>();
for (Map.Entry<String, String> e : oldProp.entrySet()) {
String key = e.getKey();
String oldVal = e.getValue();
String newVal = newProp.get(key);
if (newVal == null) {
oldProp.remove(key);
}
tmp.put(key, new ThreeTuple<>(key, oldVal, newVal));
}
for (Map.Entry<String, String> e : newProp.entrySet()) {
String key = e.getKey();
String newVal = e.getValue();
String oldVal = oldProp.putIfAbsent(key, newVal);
tmp.putIfAbsent(key, new ThreeTuple<>(key, oldVal, newVal));
}
data.addAll(new ArrayList<>(tmp.values()));
oldProp.clear();
oldProp.putAll(newProp);
}
private void onDataChange(List<ThreeTuple<String, String, String>> data) {
for (ThreeTuple<String, String, String> tuple : data) {
for (ConfigListener listener : listeners) {
if (!tuple.valueEquals()) {
listener.onChange(tuple.k1, tuple.v1, tuple.v2);
}
}
}
}
}
}

59
tunnel-server/src/main/java/com/hellobike/base/tunnel/filter/ColumnFilter.java Normal file
View File

@ -0,0 +1,59 @@
package com.hellobike.base.tunnel.filter;
import com.hellobike.base.tunnel.model.ColumnData;
import com.hellobike.base.tunnel.model.Event;
import java.util.List;
import java.util.stream.Collectors;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-10-30
*/
public class ColumnFilter implements IEventFilter {
private final List<String> columnNames;
public ColumnFilter(List<String> columnNames) {
this.columnNames = columnNames;
}
@Override
public boolean filter(Event event) {
if (columnNames == null || columnNames.isEmpty()) {
return true;
}
List<ColumnData> dataList = event.getDataList()
.stream()
.filter(cd -> {
String name = cd.getName();
return columnNames
.stream()
.anyMatch(s -> s.contains(name));
}).collect(Collectors.toList());
if (dataList.isEmpty()) {
return false;
}
return true;
}
}

45
tunnel-server/src/main/java/com/hellobike/base/tunnel/filter/EventTypeFilter.java Normal file
View File

@ -0,0 +1,45 @@
package com.hellobike.base.tunnel.filter;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.model.EventType;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-10-30
*/
public class EventTypeFilter implements IEventFilter {
private final EventType eventType;
public EventTypeFilter(EventType eventType) {
this.eventType = eventType;
}
@Override
public boolean filter(Event event) {
if (event == null) {
return false;
}
return getEventType() == null || event.getEventType() == getEventType();
}
public EventType getEventType() {
return eventType;
}
}

32
tunnel-server/src/main/java/com/hellobike/base/tunnel/filter/IEventFilter.java Normal file
View File

@ -0,0 +1,32 @@
package com.hellobike.base.tunnel.filter;
import com.hellobike.base.tunnel.model.Event;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-10-25
*/
public interface IEventFilter {
/**
* @param event msg event
* @return whether test success
*/
boolean filter(Event event);
}

43
tunnel-server/src/main/java/com/hellobike/base/tunnel/filter/SchemaFilter.java Normal file
View File

@ -0,0 +1,43 @@
package com.hellobike.base.tunnel.filter;
import com.hellobike.base.tunnel.model.Event;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-10-30
*/
public class SchemaFilter implements IEventFilter {
private final String schemaName;
public SchemaFilter(String schemaName) {
this.schemaName = schemaName;
}
@Override
public boolean filter(Event event) {
if (event == null || event.getSchema() == null) {
return false;
}
return getSchemaName() == null || getSchemaName().contains(event.getSchema());
}
public String getSchemaName() {
return schemaName;
}
}

47
tunnel-server/src/main/java/com/hellobike/base/tunnel/filter/TableNameFilter.java Normal file
View File

@ -0,0 +1,47 @@
package com.hellobike.base.tunnel.filter;
import com.hellobike.base.tunnel.model.Event;
import org.apache.commons.lang3.StringUtils;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-10-30
*/
public class TableNameFilter implements IEventFilter {
private final String tableNameExp;
private final Pattern pattern;
public TableNameFilter(String tableNameExp) {
this.tableNameExp = StringUtils.isBlank(tableNameExp) ? ".*" : tableNameExp;
this.pattern = Pattern.compile(this.tableNameExp);
}
@Override
public boolean filter(Event event) {
if (event == null || event.getTable() == null) {
return false;
}
Matcher matcher = pattern.matcher(event.getTable());
return matcher.matches();
}
}

48
tunnel-server/src/main/java/com/hellobike/base/tunnel/ha/DistributedLock.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.ha;
import java.util.concurrent.TimeUnit;
/**
* @author machunxiao create at 2018-12-26
*/
public interface DistributedLock {
/**
* 获取锁
*
* @throws LockingException
*/
void lock() throws LockingException;
/**
* 尝试获取锁
*
* @param timeout 超时时间
* @param unit 时间刻度
* @return lock success:true else:false
*/
boolean tryLock(long timeout, TimeUnit unit);
/**
* 释放锁
*
* @throws LockingException
*/
void unlock() throws LockingException;
}

32
tunnel-server/src/main/java/com/hellobike/base/tunnel/ha/LockingException.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.ha;
/**
* @author machunxiao create at 2018-12-26
*/
public class LockingException extends RuntimeException {
private static final long serialVersionUID = 2256598708734964597L;
public LockingException(String msg, Throwable e) {
super(msg, e);
}
public LockingException(String msg) {
super(msg);
}
}

157
tunnel-server/src/main/java/com/hellobike/base/tunnel/ha/ZkDistributedLock.java Normal file
View File

@ -0,0 +1,157 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.ha;
import com.google.common.base.Preconditions;
import org.apache.curator.framework.CuratorFramework;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* @author machunxiao create at 2018-12-26
*/
public class ZkDistributedLock implements DistributedLock {
private static final Logger LOG = LoggerFactory.getLogger(ZkDistributedLock.class.getName());
private final CuratorFramework zkClient;
private final String lockPath;
private final AtomicBoolean aborted = new AtomicBoolean(false);
private CountDownLatch syncPoint;
private boolean holdsLock = false;
private String currentId;
private String currentNode;
private String watchedNode;
/**
* Creates a distributed lock using the given {@code zkClient} to coordinate locking.
*
* @param zkClient The ZooKeeper client to use.
* @param lockPath The path used to manage the lock under.
* @param acl The acl to apply to newly created lock nodes.
*/
public ZkDistributedLock(CuratorFramework zkClient, String lockPath, Iterable<ACL> acl) {
this.zkClient = zkClient;
this.lockPath = lockPath;
this.syncPoint = new CountDownLatch(1);
}
@Override
public synchronized void lock() throws LockingException {
if (holdsLock) {
throw new LockingException("Error, already holding a lock. Call unlock first!");
}
try {
prepare();
syncPoint.await();
if (!holdsLock) {
throw new LockingException("Error, couldn't acquire the lock!");
}
} catch (InterruptedException e) {
cancelAttempt();
throw new LockingException("InterruptedException while trying to acquire lock!", e);
} catch (Exception e) {
// No need to clean up since the node wasn't created yet.
throw new LockingException("ZkException while trying to acquire lock!", e);
}
}
@Override
public synchronized boolean tryLock(long timeout, TimeUnit unit) {
if (holdsLock) {
throw new LockingException("Error, already holding a lock. Call unlock first!");
}
try {
prepare();
boolean success = syncPoint.await(timeout, unit);
if (!success) {
return false;
}
if (!holdsLock) {
throw new LockingException("Error, couldn't acquire the lock!");
}
} catch (InterruptedException e) {
cancelAttempt();
return false;
} catch (Exception e) {
// No need to clean up since the node wasn't created yet.
throw new LockingException("ZkException while trying to acquire lock!", e);
}
return true;
}
@Override
public synchronized void unlock() throws LockingException {
if (currentId == null) {
throw new LockingException("Error, neither attempting to lock nor holding a lock!");
}
Preconditions.checkNotNull(currentId);
// Try aborting!
if (!holdsLock) {
aborted.set(true);
} else {
cleanup();
}
}
private synchronized void prepare() throws Exception {
this.zkClient.checkExists().forPath(lockPath);
// Create an EPHEMERAL_SEQUENTIAL node.
this.currentNode = this.zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath(lockPath + "/member_");
// We only care about our actual id since we want to compare ourselves to siblings.
if (currentNode.contains("/")) {
currentId = currentNode.substring(currentNode.lastIndexOf("/") + 1);
}
}
private synchronized void cancelAttempt() {
cleanup();
// Bubble up failure...
holdsLock = false;
syncPoint.countDown();
}
private void cleanup() {
LOG.info("Cleaning up!");
Preconditions.checkNotNull(currentId);
try {
Stat stat = zkClient.checkExists().forPath(currentNode);
if (stat != null) {
zkClient.delete().forPath(currentNode);
} else {
LOG.info("Called cleanup but nothing to cleanup!");
}
} catch (Exception e) {
throw new RuntimeException(e);
}
holdsLock = false;
aborted.set(false);
currentId = null;
currentNode = null;
syncPoint = new CountDownLatch(1);
}
}

107
tunnel-server/src/main/java/com/hellobike/base/tunnel/ha/ZkLock.java Normal file
View File

@ -0,0 +1,107 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.ha;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.retry.RetryNTimes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.TimeUnit;
/**
* @author machunxiao 2018-11-07
*/
public class ZkLock implements DistributedLock, AutoCloseable {
private static final Logger /**/ log = LoggerFactory.getLogger(ZkLock.class);
private final CuratorFramework /**/ zkClient;
private final InterProcessMutex /**/ lock;
private final String /**/ path;
public ZkLock(String address, String path) {
this.zkClient = startZkClient(address);
this.path = path;
this.lock = new InterProcessMutex(this.zkClient, path);
}
private static CuratorFramework startZkClient(String address) {
CuratorFramework zkClient = CuratorFrameworkFactory.builder()
.connectString(address)
.connectionTimeoutMs(Integer.MAX_VALUE)
.sessionTimeoutMs(Integer.MAX_VALUE)
.retryPolicy(new RetryNTimes(5, 1000))
.build();
zkClient.start();
log.info("ZkLog ---- ZkClient Started");
return zkClient;
}
public boolean tryLock() {
return tryLock(1000, TimeUnit.MILLISECONDS);
}
@Override
public void lock() throws LockingException {
try {
lock.acquire();
} catch (Exception e) {
throw new LockingException("lock", e);
}
}
@Override
public boolean tryLock(long timeout, TimeUnit unit) {
try {
log.info("ZkLog ---- Acquiring Lock On:{}", path);
while (true) {
if (lock.acquire(timeout, unit)) {
break;
}
}
log.info("ZkLog ---- Acquired Lock On:{}", path);
return true;
} catch (Exception e) {
//
}
return false;
}
@Override
public void unlock() {
try {
lock.release();
log.info("ZkLog ---- Release Lock Done");
} catch (Exception e) {
//
}
}
@Override
public void close() {
try {
unlock();
this.zkClient.close();
log.info("ZkLog ---- ZkClient Closed");
} catch (Exception e) {
//
}
}
}

92
tunnel-server/src/main/java/com/hellobike/base/tunnel/model/ColumnData.java Normal file
View File

@ -0,0 +1,92 @@
package com.hellobike.base.tunnel.model;
import java.io.Serializable;
import java.util.Objects;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-10-25
*/
public class ColumnData implements Serializable {
private static final long serialVersionUID = 4767055418095657107L;
private String name;
private String dataType;
private String value;
public ColumnData() {
}
public ColumnData(String name, String dataType, String value) {
this.name = name;
this.dataType = dataType;
this.value = value;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getDataType() {
return dataType;
}
public void setDataType(String dataType) {
this.dataType = dataType;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ColumnData data = (ColumnData) o;
return Objects.equals(name, data.name) &&
Objects.equals(dataType, data.dataType) &&
Objects.equals(value, data.value);
}
@Override
public int hashCode() {
return Objects.hash(name, dataType, value);
}
@Override
public String toString() {
return "ColumnData{" +
"name='" + name + '\'' +
", dataType='" + dataType + '\'' +
", value=" + value +
'}';
}
}

126
tunnel-server/src/main/java/com/hellobike/base/tunnel/model/Event.java Normal file
View File

@ -0,0 +1,126 @@
package com.hellobike.base.tunnel.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-10-25
*/
public class Event implements Serializable {
private static final long serialVersionUID = 3414755790085772526L;
private long lsn;
private transient String slotName;
private transient String serverId;
private String schema;
private String table;
private EventType eventType;
private List<ColumnData> dataList = new ArrayList<>();
public String getSlotName() {
return slotName;
}
public void setSlotName(String slotName) {
this.slotName = slotName;
}
public String getServerId() {
return serverId;
}
public void setServerId(String serverId) {
this.serverId = serverId;
}
public String getSchema() {
return schema;
}
public void setSchema(String schema) {
this.schema = schema;
}
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public EventType getEventType() {
return eventType;
}
public void setEventType(EventType eventType) {
this.eventType = eventType;
}
public List<ColumnData> getDataList() {
return dataList;
}
public void setDataList(List<ColumnData> dataList) {
this.dataList = dataList;
}
public long getLsn() {
return lsn;
}
public void setLsn(long lsn) {
this.lsn = lsn;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Event event = (Event) o;
return Objects.equals(schema, event.schema) &&
Objects.equals(table, event.table) &&
eventType == event.eventType &&
Objects.equals(dataList, event.dataList);
}
@Override
public int hashCode() {
return Objects.hash(schema, table, eventType, dataList);
}
@Override
public String toString() {
return "Event{" +
"schema='" + schema + '\'' +
", table='" + table + '\'' +
", eventType=" + eventType +
", dataList=" + dataList +
'}';
}
}

49
tunnel-server/src/main/java/com/hellobike/base/tunnel/model/EventType.java Normal file
View File

@ -0,0 +1,49 @@
package com.hellobike.base.tunnel.model;
import java.io.Serializable;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-10-25
*/
public enum EventType implements Serializable {
// sql 语句类型
BEGIN,
COMMIT,
INSERT,
UPDATE,
DELETE;
private static final long serialVersionUID = 1L;
public static EventType getEventType(String message) {
if (EventType.INSERT.name().equalsIgnoreCase(message)) {
return EventType.INSERT;
} else if (EventType.DELETE.name().equalsIgnoreCase(message)) {
return EventType.DELETE;
} else if (EventType.UPDATE.name().equalsIgnoreCase(message)) {
return EventType.UPDATE;
} else if (EventType.BEGIN.name().equalsIgnoreCase(message)) {
return EventType.BEGIN;
} else if (EventType.COMMIT.name().equalsIgnoreCase(message)) {
return EventType.COMMIT;
} else {
throw new IllegalArgumentException("unsupported event:" + message);
}
}
}

122
tunnel-server/src/main/java/com/hellobike/base/tunnel/model/InvokeContext.java Normal file
View File

@ -0,0 +1,122 @@
package com.hellobike.base.tunnel.model;
import com.hellobike.base.tunnel.publisher.IPublisher;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-14
*/
public class InvokeContext {
private String serverId;
private String slotName;
private String jdbcUrl;
private String jdbcUser;
private String jdbcPass;
private long lsn;
private String message;
private String xid;
private Event event;
private IPublisher.Callback callback;
public String getServerId() {
return serverId;
}
public void setServerId(String serverId) {
this.serverId = serverId;
}
public String getSlotName() {
return slotName;
}
public void setSlotName(String slotName) {
this.slotName = slotName;
}
public String getJdbcUrl() {
return jdbcUrl;
}
public void setJdbcUrl(String jdbcUrl) {
this.jdbcUrl = jdbcUrl;
}
public String getJdbcUser() {
return jdbcUser;
}
public void setJdbcUser(String jdbcUser) {
this.jdbcUser = jdbcUser;
}
public String getJdbcPass() {
return jdbcPass;
}
public void setJdbcPass(String jdbcPass) {
this.jdbcPass = jdbcPass;
}
public long getLsn() {
return lsn;
}
public void setLsn(long lsn) {
this.lsn = lsn;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
public String getXid() {
return xid;
}
public void setXid(String xid) {
this.xid = xid;
}
public Event getEvent() {
return event;
}
public void setEvent(Event event) {
this.event = event;
}
public IPublisher.Callback getCallback() {
return callback;
}
public void setCallback(IPublisher.Callback callback) {
this.callback = callback;
}
}

43
tunnel-server/src/main/java/com/hellobike/base/tunnel/monitor/TunnelMonitorFactory.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.monitor;
/**
* @author machunxiao create at 2018-12-27
*/
public class TunnelMonitorFactory {
private static volatile TunnelExporter exporter;
private TunnelMonitorFactory() {
}
public static TunnelMonitor getTunnelMonitor() {
return exporter.getTunnelMonitor();
}
public static synchronized TunnelExporter initializeExporter(boolean useYukon, ExporterConfig config) {
if (exporter == null) {
synchronized (TunnelMonitorFactory.class) {
exporter = new PrometheusExporter(config);
}
}
return exporter;
}
}

186
tunnel-server/src/main/java/com/hellobike/base/tunnel/parse/EventParser.java Normal file
View File

@ -0,0 +1,186 @@
package com.hellobike.base.tunnel.parse;
import com.hellobike.base.tunnel.model.ColumnData;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.model.EventType;
import com.hellobike.base.tunnel.model.InvokeContext;
import com.hellobike.base.tunnel.store.MemStore;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-10-25
*/
public class EventParser implements IEventParser {
private MemStore memStore;
public EventParser() {
this.memStore = new MemStore();
}
private static boolean isBegin(String msg) {
return msg != null
&& msg.length() > 5
&& (msg.charAt(0) == 'B' || msg.charAt(0) == 'b')
&& (msg.charAt(1) == 'E' || msg.charAt(1) == 'e')
&& (msg.charAt(2) == 'G' || msg.charAt(2) == 'g')
&& (msg.charAt(3) == 'I' || msg.charAt(3) == 'i')
&& (msg.charAt(4) == 'N' || msg.charAt(4) == 'n');
}
private static boolean isCommit(String msg) {
return msg != null
&& msg.length() > 6
&& (msg.charAt(0) == 'C' || msg.charAt(0) == 'c')
&& (msg.charAt(1) == 'O' || msg.charAt(1) == 'o')
&& (msg.charAt(2) == 'M' || msg.charAt(2) == 'm')
&& (msg.charAt(3) == 'M' || msg.charAt(3) == 'm')
&& (msg.charAt(4) == 'I' || msg.charAt(4) == 'i')
&& (msg.charAt(5) == 'T' || msg.charAt(5) == 't');
}
@Override
public void parse(InvokeContext context) {
if (isBegin(context.getMessage()) || isCommit(context.getMessage())) {
return;
}
Event event = parseEvent(context.getMessage());
if (event == null) {
return;
}
context.setEvent(event);
memStore.store(context);
}
private Event parseEvent(String message) {
Event event = new Event();
Lexer lexer = new Lexer(message);
// "table"
lexer.nextToken(' ');
// schema_name
lexer.nextToken('.');
String schema = lexer.token();
// table_name
String table = lexer.nextToken(':');
lexer.skip(1);
// event_type
String eventType = lexer.nextToken(':');
event.setSchema(schema);
event.setTable(table);
event.setEventType(EventType.getEventType(eventType));
lexer.skip(1);
while (lexer.hasNext()) {
ColumnData data = new ColumnData();
String name = parseName(lexer);
if ("(no-tuple-data)".equals(name)) {
// 删除时,无主键,不能同步
return null;
}
String type = parseType(lexer);
lexer.skip(1);
String value = parseValue(lexer);
// 去除多余的符号 "'"
if (value.length() > 0 && value.charAt(0) == '\'' && value.charAt(value.length() - 1) == '\'') {
value = value.substring(1, value.length() - 1);
}
data.setName(name);
data.setDataType(type);
data.setValue(value);
event.getDataList().add(data);
}
return event;
}
private String parseName(Lexer lexer) {
if (lexer.current() == ' ') {
lexer.skip(1);
}
lexer.nextToken('[');
return lexer.token();
}
private String parseType(Lexer lexer) {
lexer.nextToken(']');
return lexer.token();
}
private String parseValue(Lexer lexer) {
if (lexer.current() == '\'') {
lexer.skip(1);
lexer.nextToken('\'');
return lexer.token();
}
lexer.nextToken(' ');
return lexer.token();
}
public void setMemStore(MemStore memStore) {
this.memStore = memStore;
}
private static class Lexer {
private final String input;
private final char[] array;
private final int length;
private int pos = 0;
private String token;
public Lexer(String input) {
this.input = input;
this.array = input.toCharArray();
this.length = this.array.length;
}
public String token() {
return token;
}
public String nextToken(char comma) {
if (pos < length) {
StringBuilder out = new StringBuilder(16);
while (pos < length && array[pos] != comma) {
out.append(array[pos]);
pos++;
}
pos++;
return token = out.toString();
}
return token = null;
}
public void skip(int skip) {
this.pos += skip;
}
public char current() {
return array[pos];
}
public boolean hasNext() {
return pos < length;
}
}
}

32
tunnel-server/src/main/java/com/hellobike/base/tunnel/parse/IEventParser.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.parse;
import com.hellobike.base.tunnel.model.InvokeContext;
/**
* @author machunxiao 2018-10-25
*/
public interface IEventParser {
/**
* 解析消息
*
* @param context 上下文信息
*/
void parse(InvokeContext context);
}

63
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/BasePublisher.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher;
import com.hellobike.base.tunnel.monitor.Statics;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* @author machunxiao 2018-11-15
*/
public abstract class BasePublisher {
public static List<Statics> mapToStatics(Map<String, Long> data) {
List<Statics> staticsList = new ArrayList<>();
for (Map.Entry<String, Long> entry : data.entrySet()) {
String k = entry.getKey();
Long c = entry.getValue();
String[] dst = k.split("@@");
Statics statics = Statics.createStatics(
"AppTunnelService",
dst[0],
dst[1],
dst[2],
c.intValue(),
dst[3],
""
);
staticsList.add(statics);
}
return staticsList;
}
protected void onSuccess(IPublisher.Callback callback) {
if (callback != null) {
callback.onSuccess();
}
}
protected void onFailure(IPublisher.Callback callback, Exception e) {
if (callback != null) {
callback.onFailure(e);
}
}
}

77
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/IPublisher.java Normal file
View File

@ -0,0 +1,77 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.model.InvokeContext;
import java.util.List;
/**
* @author machunxiao 2018-10-25
*/
public interface IPublisher extends AutoCloseable {
/**
* 消费消息
*
* @param event 事件
* @param callback 回调
*/
void publish(Event event, Callback callback);
/**
* 消费消息
*
* @param context 上下文
* @param callback 回调
*/
default void publish(InvokeContext context, Callback callback) {
context.getEvent().setLsn(context.getLsn());
context.getEvent().setServerId(context.getServerId());
context.getEvent().setSlotName(context.getSlotName());
this.publish(context.getEvent(), callback);
}
/**
* 关闭发布器
*/
@Override
default void close() {
}
/**
* 批量发布
*
* @param contexts 上下文
*/
default void publish(List<InvokeContext> contexts) {
if (contexts == null || contexts.isEmpty()) {
return;
}
contexts.forEach(ctx -> publish(ctx, null));
}
interface Callback {
void onSuccess();
void onFailure(Throwable t);
}
}

68
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/PublisherManager.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.model.InvokeContext;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author machunxiao 2018-10-25
*/
public class PublisherManager implements IPublisher {
private static final PublisherManager INSTANCE = new PublisherManager();
private Map<String, IPublisher> publishers = new ConcurrentHashMap<>();
private PublisherManager() {
}
public static PublisherManager getInstance() {
return INSTANCE;
}
@Override
public void publish(Event event, Callback callback) {
this.publishers.get(event.getSlotName()).publish(event, callback);
}
@Override
public void publish(InvokeContext context, Callback callback) {
this.publishers.get(context.getSlotName()).publish(context, callback);
}
@Override
public void close() {
this.publishers.values().forEach(IPublisher::close);
this.publishers.clear();
}
@Override
public void publish(List<InvokeContext> contexts) {
if (contexts.isEmpty()) {
return;
}
}
public void putPublisher(String slotName, IPublisher publisher) {
this.publishers.put(slotName, publisher);
}
}

403
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/es/EsPublisher.java Normal file
View File

@ -0,0 +1,403 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher.es;
import com.hellobike.base.tunnel.config.EsConfig;
import com.hellobike.base.tunnel.model.ColumnData;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.model.InvokeContext;
import com.hellobike.base.tunnel.monitor.TunnelMonitorFactory;
import com.hellobike.base.tunnel.publisher.BasePublisher;
import com.hellobike.base.tunnel.publisher.IPublisher;
import com.hellobike.base.tunnel.utils.NamedThreadFactory;
import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.handlers.MapHandler;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHost;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.DriverManager;
import java.util.*;
import java.util.concurrent.*;
import java.util.stream.Collectors;
import static com.hellobike.base.tunnel.utils.TimeUtils.sleepOneSecond;
/**
* @author machunxiao 2018-10-25
*/
public class EsPublisher extends BasePublisher implements IPublisher {
private static final Logger /**/ LOG = LoggerFactory.getLogger(EsPublisher.class);
private static final int /**/ MAX_CACHED = 10240;
private final List<EsConfig> /**/ esConfigs;
private final ThreadPoolExecutor /**/ executor;
private final LinkedBlockingQueue<Helper> /**/ requestHelperQueue;
private final RestHighLevelClient[] /**/ restClients;
private volatile boolean /**/ started;
public EsPublisher(List<EsConfig> esConfigs) {
this.esConfigs = esConfigs;
int total = 8;
this.executor = new ThreadPoolExecutor(total, total, 60L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(5000), new NamedThreadFactory("EsSendThread"));
this.restClients = new RestHighLevelClient[total];
for (int i = 0; i < total; i++) {
this.restClients[i] = newRestEsHighLevelClient();
}
this.requestHelperQueue = new LinkedBlockingQueue<>(81920);
started = true;
for (int i = 0; i < total; i++) {
this.executor.submit(new Sender(i));
}
}
@Override
public void publish(Event event, Callback callback) {
throw new UnsupportedOperationException();
}
@Override
public void publish(InvokeContext context, Callback callback) {
this.esConfigs.forEach(
esConfig -> internalPublish(context, callback, esConfig)
);
}
@Override
public void close() {
started = false;
this.executor.shutdown();
Arrays.stream(this.restClients).forEach(this::closeClosable);
LOG.info("EsPublisher Closed");
}
private void closeClosable(AutoCloseable closeable) {
if (closeable != null) {
try {
closeable.close();
} catch (Exception e) {
//
}
}
}
private void internalPublish(InvokeContext context, Callback callback, EsConfig esConfig) {
if (esConfig.getFilters() == null
|| esConfig.getFilters().isEmpty()
|| esConfig.getFilters().stream().allMatch(filter -> filter.filter(context.getEvent()))) {
sendToEs(esConfig, context, callback);
}
}
private void sendToEs(EsConfig esConfig, InvokeContext context, Callback callback) {
try {
requestHelperQueue.put(new Helper(context, esConfig, callback));
onSuccess(callback);
} catch (Exception e) {
//
LOG.error("Put Data To MemQueue Failure", e);
onFailure(callback, e);
}
if (requestHelperQueue.size() >= MAX_CACHED) {
forceFlushMemQueue(0);
}
}
private DocWriteRequest eventToRequest(EsConfig esConfig, InvokeContext context) {
DocWriteRequest req = null;
Map<String, String> values = context.getEvent().getDataList()
.stream()
.collect(Collectors.toMap(ColumnData::getName, ColumnData::getValue));
// column_name,column_name
String id = esConfig.getEsIdFieldNames()
.stream()
.map(esId -> String.valueOf(values.get(esId)))
.reduce((s1, s2) -> s1 + esConfig.getSeparator() + s2)
.orElse("");
if (StringUtils.isBlank(id)) {
return null;
}
String type = esConfig.getType();
String index = esConfig.getIndex();
switch (context.getEvent().getEventType()) {
case INSERT:
IndexRequest ir = new IndexRequest(index, type, id);
ir.opType(DocWriteRequest.OpType.CREATE);
execSQL(esConfig, context, values);
ir.source(values);
req = ir;
break;
case UPDATE:
UpdateRequest ur = new UpdateRequest(index, type, id);
execSQL(esConfig, context, values);
ur.doc(values);
req = ur;
break;
case DELETE:
DeleteRequest dr = new DeleteRequest(index, type, id);
dr.id(id);
req = dr;
break;
default:
break;
}
return req;
}
private void execSQL(EsConfig esConfig, InvokeContext context, Map<String, String> values) {
String sql = esConfig.getSql();
List<String> parameters = esConfig.getParameters();
if (sql != null && parameters != null) {
// select * from tb1 where id=@id,name=@name
for (String k : parameters) {
sql = sql.replace("?", getValue(values.get(k)));
}
// 执行sql,获得结果集
Map<String, Object> result = executeQuery(sql, context);
Map<String, String> newValues = new LinkedHashMap<>();
result.entrySet().stream()
.filter(e -> e.getValue() != null)
.forEach(e -> newValues.put(e.getKey(), String.valueOf(e.getValue())));
if (!newValues.isEmpty()) {
values.clear();
values.putAll(newValues);
}
}
}
private String getValue(Object val) {
if (val instanceof Number) {
return val.toString();
}
return '\'' + val.toString() + '\'';
}
private void syncSend(RestHighLevelClient restClient, List<DocWriteRequest> doc) {
long s = System.currentTimeMillis();
BulkRequest br = createBulkRequest(doc);
RequestOptions requestOptions = createRequestOptions();
int retry = 10;
while (retry > 0) {
try {
BulkResponse response = restClient.bulk(br, requestOptions);
long e = System.currentTimeMillis();
LOG.info("indexed doc:{},cost:{}ms,result:{}", doc.size(), e - s, response.hasFailures());
return;
} catch (Exception e) {
//
retry--;
LOG.error("Send Data To Es Occurred Error,retry:" + (10 - retry), e);
}
}
}
private void asyncSend(RestHighLevelClient restClient, List<DocWriteRequest> doc) {
BulkRequest br = createBulkRequest(doc);
RequestOptions requestOptions = createRequestOptions();
restClient.bulkAsync(br, requestOptions, new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse responses) {
}
@Override
public void onFailure(Exception e) {
}
});
}
private BulkRequest createBulkRequest(List<DocWriteRequest> doc) {
BulkRequest br = new BulkRequest();
br.add(doc);
br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
br.waitForActiveShards(ActiveShardCount.ONE);
return br;
}
private RequestOptions createRequestOptions() {
RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
builder.addHeader("Connection", "Keep-Alive");
return builder.build();
}
private List<Helper> pollHelperFromQueue(BlockingQueue<Helper> queue) {
int len = queue.size();
int capacity = Math.min(MAX_CACHED / 2, len);
List<Helper> helpers = new ArrayList<>(capacity);
queue.drainTo(helpers, capacity);
return helpers;
}
private Event findMaxEvent(List<Helper> helpers) {
if (helpers == null || helpers.isEmpty()) {
return null;
}
return helpers.stream()
.max(Comparator.comparingLong(h1 -> h1.context.getLsn()))
.map(helper -> helper.context.getEvent())
.orElse(null);
}
private List<DocWriteRequest> transferToReq(List<Helper> helpers) {
return helpers.stream()
.map(helper -> eventToRequest(helper.config, helper.context))
.filter(Objects::nonNull)
.collect(Collectors.toList());
}
private RestHighLevelClient newRestEsHighLevelClient() {
return new RestHighLevelClient(RestClient.builder(
this.esConfigs
.stream()
.map(esConfig -> HttpHost.create(esConfig.getServer()))
.toArray(HttpHost[]::new)
));
}
private void forceFlushMemQueue(int idx) {
List<Helper> helpers = pollHelperFromQueue(requestHelperQueue);
try {
if (helpers.isEmpty()) {
return;
}
syncSend(restClients[idx], transferToReq(helpers));
Event maxEvent = findMaxEvent(helpers);
if (maxEvent != null) {
LOG.info("flush queue success,lsn:{}", maxEvent.getLsn());
}
} finally {
if (!helpers.isEmpty()) {
Map<String, Long> data = getMonitorData(helpers);
mapToStatics(data).forEach(statics ->
TunnelMonitorFactory.getTunnelMonitor().collect(statics)
);
}
}
}
private Map<String, Long> getMonitorData(List<Helper> helpers) {
return helpers.stream()
.map(helper -> {
helper.context.getEvent().setSlotName(helper.context.getSlotName());
return helper.context.getEvent();
})
.collect(Collectors.groupingBy(event ->
event.getSchema() + "@@" + event.getSlotName() + "@@" + event.getTable() + "@@es", Collectors.counting()));
}
private Map<String, Object> executeQuery(String sql, InvokeContext context) {
Connection connection = null;
try {
connection = DriverManager.getConnection(context.getJdbcUrl(), context.getJdbcUser(), context.getJdbcPass());
QueryRunner qr = new QueryRunner();
Map<String, Object> query = qr.query(connection, sql, new MapHandler());
if (query == null || query.isEmpty()) {
query = new LinkedHashMap<>();
LOG.warn("Select Nothing By SQL:{}", sql);
}
return query;
} catch (Exception e) {
//
} finally {
closeClosable(connection);
}
return new LinkedHashMap<>();
}
private static class Helper {
final InvokeContext context;
final EsConfig config;
final Callback callback;
private Helper(InvokeContext context, EsConfig config, Callback callback) {
this.context = context;
this.config = config;
this.callback = callback;
}
}
private class Sender implements Runnable {
private final int idx;
public Sender(int idx) {
this.idx = idx;
}
@Override
public void run() {
while (started) {
long s = System.currentTimeMillis();
try {
forceFlushMemQueue(this.idx);
} catch (Exception e) {
LOG.warn("flush data to es failure", e);
} finally {
sleepOneSecond(s, System.currentTimeMillis());
}
}
}
}
}

358
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/hbase/HBaseClient.java Normal file
View File

@ -0,0 +1,358 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher.hbase;
import com.hellobike.base.tunnel.config.HBaseConfig;
import com.hellobike.base.tunnel.model.ColumnData;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.monitor.TunnelMonitorFactory;
import com.hellobike.base.tunnel.publisher.BasePublisher;
import com.hellobike.base.tunnel.utils.DefaultObjectPoolFactory;
import com.hellobike.base.tunnel.utils.NamedThreadFactory;
import com.hellobike.base.tunnel.utils.ObjectManager;
import com.hellobike.base.tunnel.utils.ObjectPool;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static com.hellobike.base.tunnel.utils.TimeUtils.sleepOneSecond;
/**
* @author machunxiao create at 2018-11-27
*/
public class HBaseClient {
private static final Logger LOGGER = LoggerFactory.getLogger(HBaseClient.class);
private static final int MAX_CACHE = 10240;
private static final int THREAD_NUM = 4;
private static volatile HBaseClient instance = null;
private final Configuration cfg;
private final ObjectPool<Connection> pool;
private final ArrayBlockingQueue<InsertHelper> insertQueue = new ArrayBlockingQueue<>(40960);
private final ArrayBlockingQueue<DeleteHelper> deleteQueue = new ArrayBlockingQueue<>(40960);
private final ThreadPoolExecutor insertExecutor;
private final ThreadPoolExecutor deleteExecutor;
private Map<String, Table> tables = new ConcurrentHashMap<>();
private volatile boolean started;
private HBaseClient() {
this.cfg = HBaseConfiguration.create();
this.pool = new DefaultObjectPoolFactory().createObjectPool(new ConnManager());
this.insertExecutor = new ThreadPoolExecutor(THREAD_NUM, THREAD_NUM, 60, TimeUnit.SECONDS, new ArrayBlockingQueue<>(20480), new NamedThreadFactory("HBaseInsertThread"));
this.deleteExecutor = new ThreadPoolExecutor(THREAD_NUM, THREAD_NUM, 60, TimeUnit.SECONDS, new ArrayBlockingQueue<>(20480), new NamedThreadFactory("HBaseDeleteThread"));
started = true;
for (int i = 0; i < THREAD_NUM; i++) {
this.insertExecutor.submit(new InsertTask());
this.deleteExecutor.submit(new DeleteTask());
}
}
private HBaseClient(String quorum) {
this.cfg = HBaseConfiguration.create();
this.cfg.set("hbase.zookeeper.quorum", quorum);
this.pool = new DefaultObjectPoolFactory().createObjectPool(new ConnManager());
this.insertExecutor = new ThreadPoolExecutor(THREAD_NUM, THREAD_NUM, 60, TimeUnit.SECONDS, new ArrayBlockingQueue<>(20480), new NamedThreadFactory("HBaseInsertThread"));
this.deleteExecutor = new ThreadPoolExecutor(THREAD_NUM, THREAD_NUM, 60, TimeUnit.SECONDS, new ArrayBlockingQueue<>(20480), new NamedThreadFactory("HBaseDeleteThread"));
started = true;
for (int i = 0; i < THREAD_NUM; i++) {
this.insertExecutor.submit(new InsertTask());
this.deleteExecutor.submit(new DeleteTask());
}
}
public static HBaseClient getInstance() {
if (instance == null) {
synchronized (HBaseClient.class) {
if (instance == null) {
instance = new HBaseClient();
}
}
}
return instance;
}
public static HBaseClient getInstance(String quorum) {
if (instance == null) {
synchronized (HBaseClient.class) {
if (instance == null) {
instance = new HBaseClient(quorum);
}
}
}
return instance;
}
public void insert(HBaseConfig config, Event event) {
String rowKey = generateRowKey(config, event.getDataList());
if (StringUtils.isBlank(rowKey)) {
return;
}
Put put = new Put(Bytes.toBytes(rowKey));
for (ColumnData cd : event.getDataList()) {
put.addColumn(Bytes.toBytes(config.getFamily()), Bytes.toBytes(cd.getName()), Bytes.toBytes(cd.getValue()));
}
InsertHelper ih = createInsertHelper(config, event, put);
try {
insertQueue.put(ih);
} catch (Exception ignore) {
//
}
if (insertQueue.size() >= MAX_CACHE) {
doInsert();
}
}
public void update(HBaseConfig config, Event event) {
insert(config, event);
}
public void delete(HBaseConfig config, Event event) {
String rowKey = generateRowKey(config, event.getDataList());
if (StringUtils.isBlank(rowKey)) {
return;
}
DeleteHelper dh = createDeleteHelper(config, event, rowKey);
try {
deleteQueue.put(dh);
} catch (Exception ignore) {
//
}
if (deleteQueue.size() >= MAX_CACHE) {
doDelete();
}
}
public void close() {
this.started = true;
this.doDelete();
this.doInsert();
this.insertExecutor.shutdown();
this.deleteExecutor.shutdown();
this.pool.close();
}
private DeleteHelper createDeleteHelper(HBaseConfig config, Event event, String rowKey) {
DeleteHelper dh = new DeleteHelper();
dh.table = config.getHbaseTable();
dh.schema = event.getSchema();
dh.slotName = event.getSlotName();
dh.delete = new Delete(Bytes.toBytes(rowKey));
return dh;
}
private InsertHelper createInsertHelper(HBaseConfig config, Event event, Put put) {
InsertHelper ih = new InsertHelper();
ih.table = config.getHbaseTable();
ih.schema = event.getSchema();
ih.slotName = event.getSlotName();
ih.put = put;
return ih;
}
private void doDelete() {
List<DeleteHelper> deletes = pollFromQueue(deleteQueue);
if (deletes.isEmpty()) {
return;
}
long s = System.currentTimeMillis();
Map<String, List<Delete>> data = deletes.stream()
.collect(Collectors.groupingBy(dh -> dh.table, Collectors.mapping(dh -> dh.delete, Collectors.toList())));
Connection conn = pool.borrowObject();
try {
data.forEach((tableName, deleteList) -> {
try {
Table table = getTable(tableName, conn);
table.delete(deleteList);
} catch (Exception e) {
logError(e.getMessage(), e);
}
});
long e = System.currentTimeMillis();
LOGGER.info("delete msg success.queue:{},data:{},thread:{},cost:{}ms", deleteQueue.size(), deletes.size(), Thread.currentThread().getName(), (e - s));
} finally {
pool.returnObject(conn);
monitor(deletes);
}
}
private void doInsert() {
List<InsertHelper> inserts = pollFromQueue(insertQueue);
if (inserts.isEmpty()) {
return;
}
long s = System.currentTimeMillis();
Map<String, List<Put>> data = inserts.stream()
.collect(Collectors.groupingBy(ih -> ih.table, Collectors.mapping(ih -> ih.put, Collectors.toList())));
Connection conn = pool.borrowObject();
try {
data.forEach((tableName, insertList) -> {
try {
Table table = getTable(tableName, conn);
table.put(insertList);
} catch (Exception e) {
logError(e.getMessage(), e);
}
});
long e = System.currentTimeMillis();
LOGGER.info("insert msg success.queue:{},data:{},thread:{},cost:{}ms", insertQueue.size(), inserts.size(), Thread.currentThread().getName(), (e - s));
} finally {
pool.returnObject(conn);
monitor(inserts);
}
}
private void monitor(List<? extends BaseHelper> helpers) {
Map<String, Long> map = helpers.stream().collect(Collectors.groupingBy(helper -> helper.schema + "@@" + helper.slotName + "@@" + helper.table + "@@hbase", Collectors.counting()));
BasePublisher.mapToStatics(map).forEach(statics ->
TunnelMonitorFactory.getTunnelMonitor().collect(statics)
);
}
private <T> List<T> pollFromQueue(ArrayBlockingQueue<T> queue) {
int capacity = Math.min(MAX_CACHE, queue.size());
List<T> list = new ArrayList<>(capacity);
queue.drainTo(list, capacity);
return list;
}
private Table getTable(String tableName, Connection conn) throws IOException {
Table table = tables.get(tableName);
if (table == null) {
tables.putIfAbsent(tableName, getOrCreateTable(tableName, conn));
table = tables.get(tableName);
}
return table;
}
private String generateRowKey(HBaseConfig config, List<ColumnData> data) {
Map<String, String> columnKeyVal = data.stream().collect(Collectors.toMap(ColumnData::getName, ColumnData::getValue));
return config.getHbaseKey().stream()
.map(columnKeyVal::get)
.filter(Objects::nonNull)
.reduce(((s1, s2) -> s1 + "_" + s2))
.orElse(null);
}
private synchronized Table getOrCreateTable(String tableName, Connection conn) throws IOException {
TableName tb = TableName.valueOf(tableName);
return conn.getTable(tb);
}
private void logError(String msg, Throwable t) {
LOGGER.error(msg, t);
}
private Connection getConnection() throws IOException {
return ConnectionFactory.createConnection(cfg);
}
private class ConnManager implements ObjectManager<Connection> {
@Override
public Connection newInstance() throws Exception {
return getConnection();
}
@Override
public void releaseInstance(Connection connection) {
try {
connection.close();
} catch (IOException e) {
//
}
}
@Override
public boolean validateInstance(Connection connection) {
return !connection.isClosed();
}
}
private abstract class BaseHelper {
String table;
String schema;
String slotName;
}
private class InsertHelper extends BaseHelper {
Put put;
}
private class DeleteHelper extends BaseHelper {
Delete delete;
}
private class InsertTask implements Runnable {
@Override
public void run() {
while (started) {
long s = System.currentTimeMillis();
try {
doInsert();
} catch (Throwable e) {
logError("InsertTask Occurred Error", e);
} finally {
sleepOneSecond(s, System.currentTimeMillis());
}
}
}
}
private class DeleteTask implements Runnable {
@Override
public void run() {
while (started) {
long s = System.currentTimeMillis();
try {
doDelete();
} catch (Throwable e) {
logError("DeleteTask Occurred Error", e);
} finally {
sleepOneSecond(s, System.currentTimeMillis());
}
}
}
}
}

78
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/hbase/HBasePublisher.java Normal file
View File

@ -0,0 +1,78 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher.hbase;
import com.hellobike.base.tunnel.config.HBaseConfig;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.publisher.BasePublisher;
import com.hellobike.base.tunnel.publisher.IPublisher;
import java.util.List;
import java.util.Objects;
/**
* @author machunxiao create at 2018-11-27
*/
public class HBasePublisher extends BasePublisher implements IPublisher {
private final HBaseClient hBaseClient;
private final List<HBaseConfig> configs;
public HBasePublisher(List<HBaseConfig> configs) {
this.hBaseClient = getHBaseClient(configs);
this.configs = configs;
}
@Override
public void publish(Event event, Callback callback) {
for (HBaseConfig config : configs) {
if (!config.getFilters().stream().allMatch(filter -> filter.filter(event))) {
continue;
}
switch (event.getEventType()) {
case INSERT:
hBaseClient.insert(config, event);
break;
case UPDATE:
hBaseClient.update(config, event);
break;
case DELETE:
hBaseClient.delete(config, event);
break;
default:
break;
}
}
}
@Override
public void close() {
this.hBaseClient.close();
}
private HBaseClient getHBaseClient(List<HBaseConfig> configs) {
String quorum = configs.stream()
.map(HBaseConfig::getQuorum)
.filter(Objects::nonNull)
.findFirst().orElse(null);
return quorum == null ? HBaseClient.getInstance() : HBaseClient.getInstance(quorum);
}
}

71
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/hdfs/HdfsClient.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher.hdfs;
import com.alibaba.fastjson.JSON;
import com.hellobike.base.tunnel.model.Event;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.IOException;
/**
* @author machunxiao create at 2018-11-30
*/
public class HdfsClient {
private Configuration configuration;
private String fileName;
public void append(HdfsConfig config, Event event) {
try {
Configuration hadoopConfig = new Configuration();
FileSystem fileSystem = FileSystem.get(hadoopConfig);
Path hdfsPath = new Path(fileName);
FSDataOutputStream fileOutputStream = null;
try {
if (fileSystem.exists(hdfsPath)) {
fileOutputStream = fileSystem.append(hdfsPath);
} else {
fileOutputStream = fileSystem.create(hdfsPath);
}
fileOutputStream.writeUTF(JSON.toJSONString(event));
} finally {
if (fileSystem != null) {
fileSystem.close();
}
if (fileOutputStream != null) {
fileOutputStream.close();
}
}
} catch (IOException e) {
}
}
public void delete(HdfsConfig config, Event event) {
}
public void update(HdfsConfig config, Event event) {
}
}

23
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/hdfs/HdfsConfig.java Normal file
View File

@ -0,0 +1,23 @@
package com.hellobike.base.tunnel.publisher.hdfs;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-11-30
*/
public class HdfsConfig {
}

63
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/hdfs/HdfsPublisher.java Normal file
View File

@ -0,0 +1,63 @@
package com.hellobike.base.tunnel.publisher.hdfs;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.publisher.BasePublisher;
import com.hellobike.base.tunnel.publisher.IPublisher;
import java.util.List;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-11-30
*/
public class HdfsPublisher extends BasePublisher implements IPublisher {
private List<HdfsConfig> configs;
private HdfsClient hdfsClient;
public HdfsPublisher(List<HdfsConfig> configs) {
this.configs = configs;
this.hdfsClient = new HdfsClient();
}
@Override
public void publish(Event event, Callback callback) {
for (HdfsConfig config : configs) {
switch (event.getEventType()) {
case INSERT:
this.hdfsClient.append(config, event);
break;
case DELETE:
this.hdfsClient.delete(config, event);
break;
case UPDATE:
this.hdfsClient.update(config, event);
break;
default:
break;
}
}
}
@Override
public void close() {
}
}

89
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/hive/HiveClient.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher.hive;
import com.alibaba.druid.pool.DruidDataSource;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.utils.MsgUtils;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
/**
* @author machunxiao create at 2018-11-28
*/
public class HiveClient implements AutoCloseable {
private final DruidDataSource dataSource;
private final HiveConfig hiveConfig;
public HiveClient(HiveConfig hiveConfig) {
this.hiveConfig = hiveConfig;
this.dataSource = new DruidDataSource();
initDataSourceConfig();
}
public void insert(Event event) {
try (Connection conn = getConnection();
Statement stmt = conn.createStatement()) {
String sql = MsgUtils.toInsert(event);
stmt.execute(sql);
} catch (SQLException e) {
//
}
}
public void update(Event event) {
try (Connection conn = getConnection();
Statement stmt = conn.createStatement()) {
String sql = MsgUtils.toUpdate(event, this.hiveConfig.getPks());
stmt.execute(sql);
} catch (SQLException e) {
//
}
}
public void delete(Event event) {
try (Connection conn = getConnection();
Statement stmt = conn.createStatement()) {
String sql = MsgUtils.toDelete(event, this.hiveConfig.getPks());
stmt.execute(sql);
} catch (SQLException e) {
//
}
}
@Override
public void close() {
this.dataSource.close();
}
private Connection getConnection() throws SQLException {
return dataSource.getConnection();
}
private void initDataSourceConfig() {
this.dataSource.setUsername(this.hiveConfig.getUsername());
this.dataSource.setPassword(this.hiveConfig.getPassword());
this.dataSource.setUrl(this.hiveConfig.getHiveUrl());
this.dataSource.setValidationQuery("select 1");
}
}

66
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/hive/HiveConfig.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher.hive;
import java.util.List;
/**
* @author machunxiao create at 2018-11-27
*/
public class HiveConfig {
private String hiveUrl;
private String username;
private String password;
private String table;
private List<String> pks;
public String getHiveUrl() {
return hiveUrl;
}
public void setHiveUrl(String hiveUrl) {
this.hiveUrl = hiveUrl;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public List<String> getPks() {
return pks;
}
public void setPks(List<String> pks) {
this.pks = pks;
}
}

59
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/hive/HivePublisher.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher.hive;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.publisher.BasePublisher;
import com.hellobike.base.tunnel.publisher.IPublisher;
/**
* @author machunxiao create at 2018-11-27
*/
public class HivePublisher extends BasePublisher implements IPublisher {
private final HiveClient hiveClient;
public HivePublisher(HiveConfig hiveConfig) {
this.hiveClient = new HiveClient(hiveConfig);
}
@Override
public void publish(Event event, Callback callback) {
//
switch (event.getEventType()) {
case INSERT:
hiveClient.insert(event);
break;
case DELETE:
hiveClient.delete(event);
break;
case UPDATE:
hiveClient.update(event);
break;
default:
break;
}
}
@Override
public void close() {
hiveClient.close();
}
}

133
tunnel-server/src/main/java/com/hellobike/base/tunnel/publisher/kafka/KafkaPublisher.java Normal file
View File

@ -0,0 +1,133 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher.kafka;
import com.alibaba.fastjson.JSON;
import com.hellobike.base.tunnel.config.KafkaConfig;
import com.hellobike.base.tunnel.model.ColumnData;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.monitor.Statics;
import com.hellobike.base.tunnel.monitor.TunnelMonitorFactory;
import com.hellobike.base.tunnel.publisher.BasePublisher;
import com.hellobike.base.tunnel.publisher.IPublisher;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
/**
* @author machunxiao 2018-10-25
*/
public class KafkaPublisher extends BasePublisher implements IPublisher {
private static final Logger /**/ log = LoggerFactory.getLogger(KafkaPublisher.class);
private final List<KafkaConfig> /**/ kafkaConfigs;
private final KafkaProducer<String, String> /**/ producer;
public KafkaPublisher(List<KafkaConfig> kafkaConfigs) {
this.kafkaConfigs = kafkaConfigs;
this.producer = new KafkaProducer<>(getProperties(kafkaConfigs));
}
private static Properties getProperties(List<KafkaConfig> kafkaConfigs) {
KafkaConfig kafkaConfig = kafkaConfigs.get(0);
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaConfig.getServer());
props.put(ProducerConfig.ACKS_CONFIG, kafkaConfig.getAckConfig());
props.put(ProducerConfig.RETRIES_CONFIG, 0);
props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
props.put(ProducerConfig.LINGER_MS_CONFIG, 1);
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, kafkaConfig.getKeySerializer());
props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, kafkaConfig.getValSerializer());
props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, "org.apache.kafka.clients.producer.internals.DefaultPartitioner");
return props;
}
@Override
public void publish(Event event, Callback callback) {
this.kafkaConfigs.forEach(kafkaConfig -> internalPublish(kafkaConfig, event, callback));
}
@Override
public void close() {
this.producer.close();
log.info("KafkaPublisher Closed");
}
private void internalPublish(KafkaConfig kafkaConfig, Event event, Callback callback) {
if (kafkaConfig.getFilters() == null
|| kafkaConfig.getFilters().isEmpty()
|| kafkaConfig.getFilters().stream().allMatch(filter -> filter.filter(event))) {
sendToKafka(kafkaConfig, event, callback);
}
}
private void sendToKafka(KafkaConfig kafkaConfig, Event event, Callback callback) {
String value = JSON.toJSONString(event);
ProducerRecord<String, String> record = new ProducerRecord<>(kafkaConfig.getTopic(), getPrimaryKey(kafkaConfig, event), value);
String[] errors = new String[1];
try {
if (callback == null) {
producer.send(record);
} else {
producer.send(record,
(metadata, exception) -> {
if (exception != null) {
errors[0] = exception.getMessage();
KafkaPublisher.this.onFailure(callback, exception);
} else {
KafkaPublisher.this.onSuccess(callback);
}
});
}
} catch (Exception e) {
errors[0] = e.getMessage();
throw new RuntimeException(e);
} finally {
Statics statics = Statics.createStatics(
"AppTunnelService",
event.getSchema(),
event.getSlotName(),
event.getTable(),
1,
"kafka",
errors[0]
);
TunnelMonitorFactory.getTunnelMonitor().collect(statics);
}
}
private String getPrimaryKey(KafkaConfig kafkaConfig, Event event) {
List<String> pkNames = kafkaConfig.getPkNames();
if (pkNames == null || pkNames.isEmpty()) {
return null;
}
Map<String, String> data = event.getDataList().stream().collect(Collectors.toMap(ColumnData::getName, ColumnData::getValue));
return kafkaConfig.getPkNames().stream().map(data::get).reduce((s1, s2) -> s1 + "_" + s2).orElse(null);
}
}

33
tunnel-server/src/main/java/com/hellobike/base/tunnel/store/IStore.java Normal file
View File

@ -0,0 +1,33 @@
package com.hellobike.base.tunnel.store;
import com.hellobike.base.tunnel.model.InvokeContext;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao 2018-10-25
*/
public interface IStore {
/**
* 一次sql事件
*
* @param ctx 上下文
*/
void store(InvokeContext ctx);
}

147
tunnel-server/src/main/java/com/hellobike/base/tunnel/store/MemStore.java Normal file
View File

@ -0,0 +1,147 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.store;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.model.InvokeContext;
import com.hellobike.base.tunnel.publisher.PublisherManager;
import com.hellobike.base.tunnel.utils.NamedThreadFactory;
import com.hellobike.base.tunnel.utils.TimeUtils;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* @author machunxiao 2018-10-25
*/
public class MemStore implements IStore, AutoCloseable {
private static final int PER_CONTEXTS = 10240;
private static final int MAX_CONTEXTS = PER_CONTEXTS * 5;
private static final int CPU_NUMBERS = Runtime.getRuntime().availableProcessors();
private static final int THD_NUMBERS = CPU_NUMBERS << 1;
private final AtomicBoolean started = new AtomicBoolean(Boolean.FALSE);
private final Map<String, BlockingQueue<InvokeContext>> caches;
private final ThreadPoolExecutor executor;
public MemStore() {
this.caches = new ConcurrentHashMap<>();
this.executor = new ThreadPoolExecutor(
THD_NUMBERS, THD_NUMBERS,
60, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(MAX_CONTEXTS),
new NamedThreadFactory("")
);
this.started.compareAndSet(Boolean.FALSE, Boolean.TRUE);
for (int i = 0; i < THD_NUMBERS; i++) {
this.executor.submit(new ConsumeTask());
}
}
private static boolean eventIsEmpty(Event event) {
return event == null ||
(event.getSchema() == null
&& event.getTable() == null
&& event.getEventType() == null
&& (event.getDataList() == null || event.getDataList().isEmpty()));
}
private static boolean ctxIsEmpty(InvokeContext ctx) {
return ctx == null || eventIsEmpty(ctx.getEvent());
}
@Override
public void store(InvokeContext ctx) {
if (ctxIsEmpty(ctx)) {
return;
}
PublisherManager.getInstance().publish(ctx, null);
}
@Override
public void close() {
this.started.compareAndSet(Boolean.TRUE, Boolean.FALSE);
this.executor.shutdown();
this.caches.values().forEach(BlockingQueue::clear);
this.caches.values().clear();
this.caches.clear();
}
public void asyncPublish(InvokeContext ctx) {
if (ctxIsEmpty(ctx)) {
return;
}
BlockingQueue<InvokeContext> queue = putToCache(ctx);
if (queue.size() > PER_CONTEXTS) {
forceFlushMemory(queue);
}
}
private synchronized BlockingQueue<InvokeContext> putToCache(InvokeContext context) {
BlockingQueue<InvokeContext> blockingQueue = caches.get(context.getSlotName());
if (blockingQueue == null) {
caches.put(context.getSlotName(), new LinkedBlockingQueue<>(MAX_CONTEXTS));
blockingQueue = caches.get(context.getSlotName());
}
try {
blockingQueue.put(context);
} catch (Exception e) {
//
}
return blockingQueue;
}
private void forceFlushMemory(BlockingQueue<InvokeContext> queue) {
if (queue.isEmpty()) {
return;
}
int capacity = started.get() ? Math.min(PER_CONTEXTS, queue.size()) : queue.size();
List<InvokeContext> contexts = new LinkedList<>();
queue.drainTo(contexts, capacity);
PublisherManager.getInstance().publish(contexts);
}
private class ConsumeTask implements Runnable {
@Override
public void run() {
while (started.get()) {
long s = System.currentTimeMillis();
try {
for (BlockingQueue<InvokeContext> queue : caches.values()) {
forceFlushMemory(queue);
}
} finally {
long e = System.currentTimeMillis();
TimeUtils.sleepOneSecond(s, e);
}
}
for (BlockingQueue<InvokeContext> queue : caches.values()) {
forceFlushMemory(queue);
}
}
}
}

99
tunnel-server/src/main/java/com/hellobike/base/tunnel/utils/CommonPool.java Normal file
View File

@ -0,0 +1,99 @@
package com.hellobike.base.tunnel.utils;
import org.apache.commons.pool2.BasePooledObjectFactory;
import org.apache.commons.pool2.PooledObject;
import org.apache.commons.pool2.impl.DefaultPooledObject;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-03
*/
public class CommonPool<T> implements ObjectPool<T> {
private final ObjectManager<T> manager;
private final GenericObjectPool<T> pool;
public CommonPool(ObjectManager<T> manager) {
this.manager = manager;
this.pool = newObjectPool();
}
@Override
public T borrowObject() {
try {
return pool.borrowObject();
} catch (Exception e) {
throw new UnsupportedOperationException(e);
}
}
@Override
public void returnObject(T obj) {
pool.returnObject(obj);
}
@Override
public void close() {
this.pool.close();
}
private GenericObjectPool<T> newObjectPool() {
GenericObjectPool<T> pool = new GenericObjectPool<>(new CommonFactory()); // NOSONAR
pool.setConfig(newPoolConfig());
return pool;
}
private GenericObjectPoolConfig<T> newPoolConfig() {
GenericObjectPoolConfig<T> config = new GenericObjectPoolConfig<>();
config.setTestWhileIdle(true);
config.setTestOnCreate(true);
config.setTestOnBorrow(true);
config.setTestOnReturn(false);
config.setMaxTotal(100);
config.setMinIdle(30);
config.setMaxIdle(80);
return config;
}
private class CommonFactory extends BasePooledObjectFactory<T> {
@Override
public T create() throws Exception {
return manager.newInstance();
}
@Override
public PooledObject<T> wrap(T t) {
return new DefaultPooledObject<>(t);
}
@Override
public boolean validateObject(PooledObject<T> p) {
return manager.validateInstance(p.getObject());
}
@Override
public void destroyObject(PooledObject<T> p) {
manager.releaseInstance(p.getObject());
}
}
}

29
tunnel-server/src/main/java/com/hellobike/base/tunnel/utils/DefaultObjectPoolFactory.java Normal file
View File

@ -0,0 +1,29 @@
package com.hellobike.base.tunnel.utils;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-05
*/
public class DefaultObjectPoolFactory implements ObjectPoolFactory {
@Override
public <T> ObjectPool<T> createObjectPool(ObjectManager<T> manager) {
return new CommonPool<>(manager);
}
}

53
tunnel-server/src/main/java/com/hellobike/base/tunnel/utils/FileUtils.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.utils;
import com.hellobike.base.tunnel.TunnelLauncher;
import java.io.FileInputStream;
import java.io.InputStream;
import java.nio.file.Paths;
/**
* @author machunxiao create at 2018-12-26
*/
public class FileUtils {
private FileUtils() {
}
public static InputStream load(String file) {
InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(file);
if (is == null) {
is = TunnelLauncher.class.getResourceAsStream(file);
}
if (is == null) {
is = loadFromFileSystem(file);
}
return is;
}
private static InputStream loadFromFileSystem(String configPath) {
if (Paths.get(configPath).toFile().exists()) {
try {
return new FileInputStream(Paths.get(configPath).toFile());
} catch (Exception ignore) {
//
}
}
return null;
}
}

74
tunnel-server/src/main/java/com/hellobike/base/tunnel/utils/MsgUtils.java Normal file
View File

@ -0,0 +1,74 @@
package com.hellobike.base.tunnel.utils;
import com.hellobike.base.tunnel.model.Event;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.List;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-10
*/
public class MsgUtils {
private MsgUtils() {
}
public static String toUpdate(Event event, List<String> pkNames) {
String schema = event.getSchema();
String table = event.getTable();
String kv = event.getDataList().stream().map(cd -> cd.getName() + "=" + cd.getValue()).reduce((s1, s2) -> s1 + "," + s2).orElse("");
String pk = getPKValues(event, pkNames);
return "update " + schema + "." + table + " set " + pk + " where " + kv;
}
public static String toDelete(Event event, List<String> pkNames) {
String schema = event.getSchema();
String table = event.getTable();
String pk = getPKValues(event, pkNames);
return "delete from " + schema + "." + table + " where " + pk;
}
public static String toInsert(Event event) {
String schema = event.getSchema();
String table = event.getTable();
List<String> keys = new ArrayList<>();
List<String> vals = new ArrayList<>();
event.getDataList().forEach(cd -> {
keys.add(cd.getName());
vals.add("'" + cd.getValue() + "'");
});
return "insert into " + schema + "." + table + "(" + StringUtils.join(keys, ',') + ") values(" + StringUtils.join(vals, ',') + ")";
}
private static String getPKValues(Event event, List<String> pkNames) {
return event.getDataList().stream()
.filter(cd -> pkNames.contains(cd.getName()))
.map(cd -> cd.getName() + "=" + cd.getValue())
.reduce((s1, s2) -> s1 + "," + s2)
.orElse("");
}
}

38
tunnel-server/src/main/java/com/hellobike/base/tunnel/utils/NamedThreadFactory.java Normal file
View File

@ -0,0 +1,38 @@
package com.hellobike.base.tunnel.utils;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-11-23
*/
public class NamedThreadFactory implements ThreadFactory {
private final AtomicInteger index = new AtomicInteger();
private String name;
public NamedThreadFactory(String name) {
this.name = name;
}
@Override
public Thread newThread(Runnable r) {
return new Thread(r, this.name + "-" + index.incrementAndGet());
}
}

45
tunnel-server/src/main/java/com/hellobike/base/tunnel/utils/ObjectManager.java Normal file
View File

@ -0,0 +1,45 @@
package com.hellobike.base.tunnel.utils;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-03
*/
public interface ObjectManager<T> {
/**
* 产生一个实例
*
* @return 实例
* @throws Exception 错误信息
*/
T newInstance() throws Exception;
/**
* 释放资源
*
* @param instance 实例
*/
default void releaseInstance(T instance) {
}
default boolean validateInstance(T instance) {
return true;
}
}

45
tunnel-server/src/main/java/com/hellobike/base/tunnel/utils/ObjectPool.java Normal file
View File

@ -0,0 +1,45 @@
package com.hellobike.base.tunnel.utils;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-03
*/
public interface ObjectPool<T> extends AutoCloseable {
/**
* get a object from pool
*
* @return instance
*/
T borrowObject();
/**
* return obj to pool
*
* @param obj object
* @return
*/
void returnObject(T obj);
/**
* 释放资源
*/
@Override
void close();
}

27
tunnel-server/src/main/java/com/hellobike/base/tunnel/utils/ObjectPoolFactory.java Normal file
View File

@ -0,0 +1,27 @@
package com.hellobike.base.tunnel.utils;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-05
*/
public interface ObjectPoolFactory {
<T> ObjectPool<T> createObjectPool(ObjectManager<T> manager);
}

43
tunnel-server/src/main/java/com/hellobike/base/tunnel/utils/TimeUtils.java Normal file
View File

@ -0,0 +1,43 @@
package com.hellobike.base.tunnel.utils;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao
*/
public class TimeUtils {
private TimeUtils() {
}
public static void sleepInMills(long mills) {
if (mills <= 0) {
return;
}
try {
Thread.sleep(mills);
} catch (InterruptedException e) {
//
}
}
public static void sleepOneSecond(long s, long e) {
long cost = 1000 + s - e;
TimeUtils.sleepInMills(cost);
}
}

1
tunnel-server/src/main/resources/META-INF/app.properties Normal file
View File

@ -0,0 +1 @@
app.id=AppTunnelService

1
tunnel-server/src/main/resources/conf/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
*.properties

30
tunnel-server/src/main/resources/log4j2.xml Normal file
View File

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- copy from sprint boot -->
<Configuration status="warn" shutdownHook="disable">
<Properties>
<Property name="PID">????</Property>
<Property name="LOG_LEVEL_PATTERN">%5p</Property>
<Property name="CONSOLE_LOG_PATTERN">[%d{yyyy-MM-dd HH:mm:ss.SSS}] [${LOG_LEVEL_PATTERN}] [%20.20t] [%-40.40c{1.}] %m%n</Property>
<Property name="LOG_HOME">logs</Property>
</Properties>
<Appenders>
<Console name="Console" target="SYSTEM_OUT" follow="true">
<PatternLayout pattern="${sys:CONSOLE_LOG_PATTERN}"/>
</Console>
<File name="LogFile" fileName="logs/tunnel.log">
<ThresholdFilter level="error" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout>
<Pattern>${sys:CONSOLE_LOG_PATTERN}</Pattern>
</PatternLayout>
</File>
<Async name="Async" BufferSize="65536">
<AppenderRef ref="LogFile"/>
</Async>
</Appenders>
<Loggers>
<AsyncRoot level="info">
<AppenderRef ref="Console"/>
<AppenderRef ref="Async"/>
</AsyncRoot>
</Loggers>
</Configuration>

7
tunnel-server/src/main/resources/yukon.properties Normal file
View File

@ -0,0 +1,7 @@
yukon.appender.a2.name=kafkaAppender
yukon.appender.a2.class=com.hellobike.yukon.kafka11.appender.KafkaAppender
yukon.appender.a2.topic=pg_transfer_monitor-dev
yukon.appender.a2.brokers=10.111.70.58:9092,10.111.70.59:9092,10.111.70.60:9092
yukon.logger.l1.name=TunnelLogger
yukon.logger.l1.level=info
yukon.logger.l1.appenderRef=kafkaAppender

72
tunnel-server/src/test/java/com/hellobike/base/tunnel/TunnelServerTest.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel;
import com.hellobike.base.tunnel.utils.TimeUtils;
import org.junit.Test;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* @author machunxiao create at 2018-12-05
*/
public class TunnelServerTest {
private static volatile boolean stopped = false;
@Test
public void test_threadPool() {
int total = 4;
ThreadPoolExecutor executor = new ThreadPoolExecutor(total, total, 60, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100));
for (int i = 0; i < total; i++) {
executor.submit(new Task(i));
}
try {
executor.awaitTermination(1, TimeUnit.SECONDS);
} catch (Exception e) {
//
}
stopped = true;
executor.shutdown();
}
private static class Task implements Runnable {
private final int idx;
private Task(int idx) {
this.idx = idx;
}
@Override
public void run() {
while (!stopped) {
TimeUtils.sleepInMills(1000);
}
}
@Override
public String toString() {
return "Task{" +
"idx=" + idx +
'}';
}
}
}

61
tunnel-server/src/test/java/com/hellobike/base/tunnel/config/file/FileConfigLoaderTest.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.config.file;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
import java.nio.file.Paths;
/**
* @author machunxiao create at 2018-12-26
*/
public class FileConfigLoaderTest {
private static final Logger log = LoggerFactory.getLogger(FileConfigLoaderTest.class);
@Test
public void testFileConfigLoader() {
String file = "test_config.properties";
FileConfigLoader loader = new FileConfigLoader(getFilePath(file));
loader.addChangeListener((key, oldValue, newValue) -> log.info("key:{},old:{},new:{}", key, oldValue, newValue));
try {
Thread.currentThread().join(1000 * 60 * 10);
} catch (InterruptedException e) {
//
}
loader.close();
}
private String getFilePath(String file) {
URL url = Thread.currentThread().getContextClassLoader().getResource(file);
if (url == null) {
url = FileConfigLoaderTest.class.getClassLoader().getResource(file);
}
if (url == null) {
File f = Paths.get(file).toFile();
if (f.exists()) {
return f.getPath();
}
} else {
return url.getPath();
}
return null;
}
}

19
tunnel-server/src/test/java/com/hellobike/base/tunnel/config/package-info.java Normal file
View File

@ -0,0 +1,19 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-26
*/
package com.hellobike.base.tunnel.config;

119
tunnel-server/src/test/java/com/hellobike/base/tunnel/filter/FilterTest.java Normal file
View File

@ -0,0 +1,119 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.filter;
import com.hellobike.base.tunnel.model.ColumnData;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.model.EventType;
import org.junit.Assert;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @author machunxiao 2018-11-07
*/
public class FilterTest {
@Test
public void test_ColumnFilter() {
Event event = new Event();
ColumnFilter filter = new ColumnFilter(null);
boolean ret = filter.filter(event);
Assert.assertTrue(ret);
filter = new ColumnFilter(new ArrayList<>());
ret = filter.filter(event);
Assert.assertTrue(ret);
filter = new ColumnFilter(new ArrayList<>(Arrays.asList("Tom", "Age")));
List<ColumnData> columnDataList = new ArrayList<>();
event.setDataList(columnDataList);
ret = filter.filter(event);
Assert.assertFalse(ret);
ColumnData data = new ColumnData();
data.setName("om");
columnDataList.add(data);
ret = filter.filter(event);
Assert.assertTrue(ret);
}
@Test
public void test_EventTypeFilter() {
EventTypeFilter filter = new EventTypeFilter(EventType.DELETE);
boolean ret = filter.filter(null);
Assert.assertFalse(ret);
Event event = new Event();
event.setEventType(EventType.DELETE);
ret = filter.filter(event);
Assert.assertTrue(ret);
event.setEventType(EventType.INSERT);
ret = filter.filter(event);
Assert.assertFalse(ret);
}
@Test
public void test_SchemaFilter() {
SchemaFilter filter = new SchemaFilter(null);
boolean ret = filter.filter(null);
Assert.assertFalse(ret);
Event event = new Event();
ret = filter.filter(event);
Assert.assertFalse(ret);
filter = new SchemaFilter(null);
event.setSchema("Tom");
ret = filter.filter(event);
Assert.assertTrue(ret);
filter = new SchemaFilter("Tom");
event.setSchema("Tom");
ret = filter.filter(event);
Assert.assertTrue(ret);
filter = new SchemaFilter("Tom");
event.setSchema("Jack");
ret = filter.filter(event);
Assert.assertFalse(ret);
}
@Test
public void test_TableNameFilter() {
TableNameFilter filter = new TableNameFilter(null);
boolean ret = filter.filter(null);
Assert.assertFalse(ret);
Event event = new Event();
ret = filter.filter(event);
Assert.assertFalse(ret);
filter = new TableNameFilter("Tom.*");
event.setTable("Tom123");
ret = filter.filter(event);
Assert.assertTrue(ret);
}
}

92
tunnel-server/src/test/java/com/hellobike/base/tunnel/ha/ZkLockTest.java Normal file
View File

@ -0,0 +1,92 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.ha;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @author machunxiao 2018-11-07
*/
public class ZkLockTest {
private static final Logger LOGGER = LoggerFactory.getLogger(ZkLockTest.class);
private static final ExecutorService EXECUTOR =
new ThreadPoolExecutor(
10, 10,
0, TimeUnit.MICROSECONDS,
new ArrayBlockingQueue<>(10),
new ThreadFactory() {
private final AtomicInteger index = new AtomicInteger();
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r, "TestLock-" + index.incrementAndGet());
t.setDaemon(false);
return t;
}
});
@Test
public void test_lock() {
ZkLock lock = new ZkLock("localhost:2181", "/tunnel/test/lock");
lock.tryLock();
lock.unlock();
int total = 10;
CountDownLatch latch = new CountDownLatch(total);
CyclicBarrier barrier = new CyclicBarrier(total);
Runnable task = () -> {
try {
barrier.await();
} catch (Exception e) {
e.printStackTrace();
}
LOGGER.info("thread:{} active", Thread.currentThread().getName());
if (lock.tryLock()) {
try {
LOGGER.info("thread:{} lock success", Thread.currentThread().getName());
Thread.sleep(10L);
} catch (Exception e) {
//
} finally {
lock.unlock();
LOGGER.info("thread:{} release lock success", Thread.currentThread().getName());
latch.countDown();
}
}
};
for (int i = 0; i < total; i++) {
EXECUTOR.submit(task);
}
try {
latch.await();
} catch (InterruptedException e) {
//
}
EXECUTOR.shutdown();
lock.close();
}
}

61
tunnel-server/src/test/java/com/hellobike/base/tunnel/parse/EventParserTest.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.parse;
import com.hellobike.base.tunnel.model.InvokeContext;
import com.hellobike.base.tunnel.store.MemStore;
import org.junit.Test;
import org.mockito.Mockito;
/**
* @author machunxiao 2018-11-07
*/
public class EventParserTest {
@Test
public void test_parse() {
EventParser parser = new EventParser();
String serverId = "server-id1";
InvokeContext ctx = new InvokeContext();
ctx.setSlotName("test_slot");
ctx.setServerId(serverId);
ctx.setLsn(1000L);
String msg = "begin 12345";
ctx.setMessage(msg);
parser.parse(ctx);
msg = "commit 12345";
ctx.setMessage(msg);
parser.parse(ctx);
msg = "table public.test_logic_table: INSERT: pk[integer]:1 name[character varying]:'previous value'";
ctx.setMessage(msg);
MemStore mock = Mockito.mock(MemStore.class);
Mockito.doNothing().when(mock).store(Mockito.any());
parser.setMemStore(mock);
parser.parse(ctx);
msg = "table public.test_logic_table: DELETE: (no-tuple-data)";
ctx.setMessage(msg);
parser.parse(ctx);
}
}

207
tunnel-server/src/test/java/com/hellobike/base/tunnel/publisher/EsPublisherTest.java Normal file
View File

@ -0,0 +1,207 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher;
import com.hellobike.base.tunnel.config.EsConfig;
import com.hellobike.base.tunnel.filter.IEventFilter;
import com.hellobike.base.tunnel.filter.TableNameFilter;
import com.hellobike.base.tunnel.model.ColumnData;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.model.EventType;
import com.hellobike.base.tunnel.model.InvokeContext;
import com.hellobike.base.tunnel.publisher.es.EsPublisher;
import com.hellobike.base.tunnel.utils.TimeUtils;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic;
import pl.allegro.tech.embeddedelasticsearch.PopularProperties;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
/**
* @author machunxiao 2018-11-08
*/
public class EsPublisherTest {
private static final Logger LOGGER = LoggerFactory.getLogger(EsPublisherTest.class);
@Test
public void test_EsPublisher() {
EmbeddedElastic elastic = EmbeddedElastic.builder()
.withElasticVersion("5.0.0")
.withSetting(PopularProperties.HTTP_PORT, 9200)
.build();
try {
elastic.start();
} catch (Exception e) {
//
}
// http://localhost:9200
String table = "";
List<IEventFilter> filters = new ArrayList<>(Collections.singleton(new TableNameFilter(table)));
EsConfig esConfig = new EsConfig();
esConfig.setServer("http://localhost:9200");
esConfig.setEsIdFieldNames(Collections.singletonList("id"));
esConfig.setPkFieldNames(Collections.singletonList("id"));
esConfig.setSchema("test1");
esConfig.setTable("t_user");
esConfig.setIndex("idx_t_u");
esConfig.setType("logs");
esConfig.setFilters(filters);
List<EsConfig> esConfigs = new ArrayList<>(Collections.singleton(esConfig));
EsPublisher publisher = new EsPublisher(esConfigs);
ColumnData cd1 = new ColumnData();
cd1.setName("id");
cd1.setDataType("integer");
cd1.setValue("1001");
ColumnData cd2 = new ColumnData();
cd2.setName("name");
cd2.setDataType("varchar");
cd2.setValue("tom hanks");
ColumnData cd3 = new ColumnData();
cd3.setName("sex");
cd3.setDataType("varchar");
cd3.setValue("mail");
ColumnData cd4 = new ColumnData();
cd4.setName("email");
cd4.setDataType("varchar");
cd4.setValue("xxx@tom.com");
List<ColumnData> dataList = new ArrayList<>(Arrays.asList(cd1, cd2, cd3, cd4));
Event event = new Event();
event.setSchema("test1");
event.setTable("t_user");
event.setEventType(EventType.INSERT);
event.setDataList(dataList);
CountDownLatch latch = new CountDownLatch(1);
InvokeContext ctx = new InvokeContext();
ctx.setEvent(event);
publisher.publish(ctx, new IPublisher.Callback() {
@Override
public void onSuccess() {
LOGGER.info("success");
latch.countDown();
}
@Override
public void onFailure(Throwable t) {
LOGGER.info("failure", t);
latch.countDown();
}
});
try {
latch.await();
} catch (InterruptedException e) {
//
}
event.setEventType(EventType.UPDATE);
CountDownLatch latch3 = new CountDownLatch(1);
publisher.publish(ctx, new IPublisher.Callback() {
@Override
public void onSuccess() {
LOGGER.info("success3");
latch3.countDown();
}
@Override
public void onFailure(Throwable t) {
LOGGER.info("failure3", t);
latch3.countDown();
}
});
event.setEventType(EventType.BEGIN);
publisher.publish(ctx, null);
event.setEventType(EventType.DELETE);
CountDownLatch latch2 = new CountDownLatch(1);
publisher.publish(ctx, new IPublisher.Callback() {
@Override
public void onSuccess() {
LOGGER.info("success2");
latch2.countDown();
}
@Override
public void onFailure(Throwable t) {
LOGGER.info("failure2", t);
latch2.countDown();
}
});
try {
latch2.await();
} catch (InterruptedException e) {
//
}
try {
elastic.stop();
} catch (Exception e) {
//
}
publisher.close();
}
@Test
public void test_BlockingQueue() {
LinkedBlockingQueue<String> queue = new LinkedBlockingQueue<>(16);
for (int i = 0; i < 16; i++) {
queue.add("idx-" + i);
}
List<String> list = new ArrayList<>();
queue.drainTo(list, 8);
Assert.assertEquals(queue.size(), list.size());
}
@Test
public void test_cost() {
int i = 0;
while (i < 10) {
long s = System.currentTimeMillis();
func1();
long e = System.currentTimeMillis();
long cost = 1000 - (e - s);
TimeUtils.sleepInMills(cost);
i++;
}
}
private void func1() {
TimeUtils.sleepInMills(100L);
LOGGER.info("called");
}
}

112
tunnel-server/src/test/java/com/hellobike/base/tunnel/publisher/KafkaPublisherTest.java Normal file
View File

@ -0,0 +1,112 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher;
import com.hellobike.base.tunnel.config.KafkaConfig;
import com.hellobike.base.tunnel.filter.IEventFilter;
import com.hellobike.base.tunnel.filter.TableNameFilter;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.publisher.kafka.KafkaPublisher;
import net.manub.embeddedkafka.EmbeddedKafka;
import net.manub.embeddedkafka.EmbeddedKafkaConfig;
import net.manub.embeddedkafka.EmbeddedKafkaConfigImpl;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.collection.immutable.HashMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
/**
* @author machunxiao 2018-11-08
*/
public class KafkaPublisherTest {
private static final Logger LOGGER = LoggerFactory.getLogger(KafkaPublisherTest.class);
@Test
public void test_KafkaPublisher() {
EmbeddedKafkaConfig cfg = new EmbeddedKafkaConfigImpl(9093, 2182, new HashMap<>(), new HashMap<>(), new HashMap<>());
EmbeddedKafka.start(cfg);
String address = "localhost:9093";
KafkaConfig kafkaConfig = new KafkaConfig();
kafkaConfig.setServer(address);
kafkaConfig.setTopic("test1");
kafkaConfig.setPartition(-1);
List<KafkaConfig> kafkaConfigs = new ArrayList<>(Collections.singleton(kafkaConfig));
String table = "";
List<IEventFilter> filters = new ArrayList<>(Collections.singleton(new TableNameFilter(table)));
kafkaConfig.setFilters(filters);
KafkaPublisher publisher = new KafkaPublisher(kafkaConfigs);
Event event = new Event();
event.setTable("t1");
CountDownLatch latch1 = new CountDownLatch(1);
IPublisher.Callback callback = new IPublisher.Callback() {
@Override
public void onSuccess() {
LOGGER.info("success");
latch1.countDown();
}
@Override
public void onFailure(Throwable t) {
LOGGER.info("failure", t);
latch1.countDown();
}
};
publisher.publish(event, callback);
try {
latch1.await();
} catch (InterruptedException e) {
//
}
publisher.publish(event, null);
table = "abc";
filters = new ArrayList<>(Collections.singleton(new TableNameFilter(table)));
kafkaConfig.setFilters(filters);
publisher = new KafkaPublisher(kafkaConfigs);
CountDownLatch latch2 = new CountDownLatch(1);
event.setTable("def");
publisher.publish(event, new IPublisher.Callback() {
@Override
public void onSuccess() {
latch2.countDown();
}
@Override
public void onFailure(Throwable t) {
LOGGER.warn("", t);
latch2.countDown();
}
});
try {
latch2.await();
} catch (InterruptedException e) {
//
}
EmbeddedKafka.stop();
}
}

58
tunnel-server/src/test/java/com/hellobike/base/tunnel/publisher/PublisherManagerTest.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher;
import com.hellobike.base.tunnel.model.Event;
import org.junit.Test;
/**
* @author machunxiao 2018-11-12
*/
public class PublisherManagerTest {
@Test
public void test_PublisherManager() {
PublisherManager.getInstance().putPublisher("test123", (event, callback) -> { /**/ });
PublisherManager.getInstance().putPublisher("test456", (event, callback) -> { /**/ });
PublisherManager.getInstance().publish((Event) null, null);
PublisherManager.getInstance().publish((Event) null, new IPublisher.Callback() {
@Override
public void onSuccess() {
}
@Override
public void onFailure(Throwable t) {
}
});
PublisherManager.getInstance().publish(new Event(), null);
PublisherManager.getInstance().publish(new Event(), new IPublisher.Callback() {
@Override
public void onSuccess() {
}
@Override
public void onFailure(Throwable t) {
}
});
}
}

90
tunnel-server/src/test/java/com/hellobike/base/tunnel/publisher/hbase/HBaseClientTest.java Normal file
View File

@ -0,0 +1,90 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher.hbase;
import com.hellobike.base.tunnel.config.HBaseConfig;
import com.hellobike.base.tunnel.model.ColumnData;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.model.EventType;
import com.hellobike.base.tunnel.utils.TimeUtils;
import org.junit.Assert;
import org.junit.Test;
import java.util.Collections;
import java.util.List;
/**
* @author machunxiao create at 2018-11-28
*/
public class HBaseClientTest {
@Test
public void test_HBaseClient() {
HBaseClient client = HBaseClient.getInstance();
Assert.assertNotNull(client);
}
@Test
public void test_HBaseClient_insert() {
HBaseClient client = HBaseClient.getInstance();
String rowKey = "number";
Event event = new Event();
event.setSchema("db1");
event.setTable("go_hbase_score");
event.setEventType(EventType.INSERT);
List<ColumnData> dataList = event.getDataList();
ColumnData cd1 = new ColumnData();
cd1.setName("number");
cd1.setValue("1123");
cd1.setDataType("integer");
dataList.add(cd1);
HBaseConfig config = new HBaseConfig();
config.setFamily("f1");
config.setQualifier("q1");
config.setHbaseTable("go_hbase_score");
config.setPks(Collections.singletonList("number"));
config.setHbaseKey(Collections.singletonList("number"));
client.insert(config, event);
TimeUtils.sleepInMills(5000L);
client.close();
}
@Test
public void test_HBaseClient_update() {
HBaseClient client = HBaseClient.getInstance();
String rowKey = "pk001";
Event event = new Event();
HBaseConfig config = new HBaseConfig();
client.update(config, event);
}
@Test
public void test_HBaseClient_delete() {
HBaseClient client = HBaseClient.getInstance();
String rowKey = "pk001";
Event event = new Event();
event.setTable("tb1");
HBaseConfig config = new HBaseConfig();
client.delete(config, event);
}
@Test
public void test_HBaseClient_get() {
}
}

56
tunnel-server/src/test/java/com/hellobike/base/tunnel/publisher/hbase/HBasePublisherTest.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher.hbase;
import com.hellobike.base.tunnel.config.HBaseConfig;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.publisher.IPublisher.Callback;
import org.junit.Test;
import java.util.Collections;
/**
* @author machunxiao create at 2018-12-05
*/
public class HBasePublisherTest {
@Test
public void test_publish() {
HBaseConfig config = new HBaseConfig();
config.setFamily("tb1_family");
config.setQualifier("tb1_qualifier");
config.setPks(Collections.singletonList("id"));
HBasePublisher publisher = new HBasePublisher(Collections.singletonList(config));
Callback callback = new EmptyCallback();
Event event = new Event();
publisher.publish(event, callback);
}
private static class EmptyCallback implements Callback {
@Override
public void onFailure(Throwable t) {
}
@Override
public void onSuccess() {
}
}
}

52
tunnel-server/src/test/java/com/hellobike/base/tunnel/publisher/hive/HiveClientTest.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.publisher.hive;
import com.hellobike.base.tunnel.model.ColumnData;
import com.hellobike.base.tunnel.model.Event;
import com.hellobike.base.tunnel.model.EventType;
import org.junit.Test;
/**
* @author machunxiao create at 2018-12-03
*/
public class HiveClientTest {
@Test
public void test_insert() {
HiveConfig config = new HiveConfig();
config.setHiveUrl("jdbc:hive2://10.111.20.161:10000/default;ssl=false;");
HiveClient hiveClient = new HiveClient(config);
hiveClient.insert(newEvent());
}
@Test
public void test_find() {
}
private Event newEvent() {
Event e = new Event();
e.setTable("test1");
e.setEventType(EventType.INSERT);
e.setSchema("default");
e.getDataList().add(new ColumnData("user_id", "string", "1001"));
e.getDataList().add(new ColumnData("device_id", "string", "20001"));
e.getDataList().add(new ColumnData("price", "double", "20001.0"));
e.getDataList().add(new ColumnData("sales", "int", "10"));
return e;
}
}

38
tunnel-server/src/test/java/com/hellobike/base/tunnel/spi/api/utils/TimeUtilsTest.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.spi.api.utils;
import com.hellobike.base.tunnel.utils.TimeUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.modules.junit4.PowerMockRunner;
/**
* @author machunxiao create at 2018-12-05
*/
@RunWith(PowerMockRunner.class)
public class TimeUtilsTest {
@Test
public void test_sleepInMills() {
TimeUtils.sleepInMills(-1L);
TimeUtils.sleepInMills(1L);
PowerMockito.spy(Thread.class);
PowerMockito.doThrow(new InterruptedException()).when(Thread.class);
TimeUtils.sleepInMills(2L);
}
}

52
tunnel-server/src/test/java/com/hellobike/base/tunnel/store/MemStoreTest.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.store;
import com.hellobike.base.tunnel.model.InvokeContext;
import com.hellobike.base.tunnel.publisher.PublisherManager;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import static org.junit.Assert.assertEquals;
/**
* @author machunxiao 2018-11-08
*/
@RunWith(PowerMockRunner.class)
@PrepareForTest(PublisherManager.class)
public class MemStoreTest {
@Test
public void test_store() {
PublisherManager mgr = Mockito.mock(PublisherManager.class);
PowerMockito.mockStatic(PublisherManager.class);
Mockito.when(PublisherManager.getInstance()).thenReturn(mgr);
PublisherManager actual = PublisherManager.getInstance();
assertEquals(mgr, actual);
MemStore memStore = new MemStore();
memStore.store(null);
InvokeContext ctx = new InvokeContext();
ctx.setSlotName("testSlotName");
memStore.store(ctx);
}
}

35
tunnel-server/src/test/resources/cfg.json Normal file
View File

@ -0,0 +1,35 @@
{
"": [
{
"address": "",
"database": "",
"username": "",
"password": "",
"slot": "",
"cluster": "",
"kafka": {
"addr": [],
"topic": "",
"filter": {
"dbName": "",
"tbName": "",
"fieldName": ""
}
},
"es": {
"addr": "",
"filter": {
"dbName": "",
"tbName": "",
"fieldName": ""
}
},
"filter": {
"dbName": "",
"tbName": "",
"fieldName": ""
}
},
{}
]
}

1
tunnel-server/src/test/resources/test_config.properties Normal file
View File

@ -0,0 +1 @@
k1=v1

23
tunnel-spi/pom.xml Normal file
View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>tunnel-all</artifactId>
<groupId>com.hellobike.base.tunnel</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>tunnel-spi</artifactId>
<packaging>pom</packaging>
<modules>
<module>tunnel-api</module>
<module>tunnel-es</module>
<module>tunnel-hbase</module>
<module>tunnel-hdfs</module>
<module>tunnel-hive</module>
<module>tunnel-kafka</module>
</modules>
</project>

15
tunnel-spi/tunnel-api/pom.xml Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>tunnel-spi</artifactId>
<groupId>com.hellobike.base.tunnel</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>tunnel-api</artifactId>
</project>

35
tunnel-spi/tunnel-api/src/main/java/com/hellobike/base/tunnel/spi/api/CellData.java Normal file
View File

@ -0,0 +1,35 @@
package com.hellobike.base.tunnel.spi.api;
import lombok.Data;
import java.io.Serializable;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-07
*/
@Data
public class CellData implements Serializable {
private static final long serialVersionUID = -3834617680446353654L;
private String name;
private String type;
private String value;
}

77
tunnel-spi/tunnel-api/src/main/java/com/hellobike/base/tunnel/spi/api/Invocation.java Normal file
View File

@ -0,0 +1,77 @@
package com.hellobike.base.tunnel.spi.api;
import lombok.Builder;
import lombok.Data;
import java.util.LinkedHashMap;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-24
*/
@Data
@Builder
public class Invocation implements Cloneable {
private String serverId;
private String slotName;
private String jdbcUrl;
private String jdbcUser;
private String jdbcPass;
private long lsn;
private String message;
private String xid;
private final LinkedHashMap<String, Object> parameters = new LinkedHashMap<>();
public void addParameter(String key, Object val) {
this.parameters.put(key, val);
}
public void removeParameter(String key) {
this.parameters.remove(key);
}
public void clear() {
this.parameters.clear();
}
@Override
protected void finalize() throws Throwable {
super.finalize();
clear();
}
@Override
protected Invocation clone() {
Invocation that = Invocation.builder()
.serverId(serverId)
.slotName(slotName)
.jdbcUrl(jdbcUrl)
.jdbcUser(jdbcUser)
.jdbcPass(jdbcPass)
.lsn(lsn)
.message(message)
.xid(xid)
.build();
that.getParameters().putAll(parameters);
return that;
}
}

36
tunnel-spi/tunnel-api/src/main/java/com/hellobike/base/tunnel/spi/api/Message.java Normal file
View File

@ -0,0 +1,36 @@
package com.hellobike.base.tunnel.spi.api;
import lombok.Data;
import java.io.Serializable;
import java.util.List;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-07
*/
@Data
public class Message implements Serializable {
private static final long serialVersionUID = 6362136769760497527L;
private String schema;
private String table;
private List<CellData> data;
}

31
tunnel-spi/tunnel-api/src/main/java/com/hellobike/base/tunnel/spi/api/OutputHandler.java Normal file
View File

@ -0,0 +1,31 @@
package com.hellobike.base.tunnel.spi.api;
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @author machunxiao create at 2018-12-07
*/
public interface OutputHandler {
/**
* 输出消息
*
* @param invocation 消息内容
*/
void handle(Invocation invocation);
}

30
tunnel-spi/tunnel-api/src/main/java/com/hellobike/base/tunnel/spi/api/TransportConfig.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Copyright 2018 Shanghai Junzheng Network Technology Co.,Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.hellobike.base.tunnel.spi.api;
/**
* @author machunxiao create at 2018-12-24
*/
public interface TransportConfig<T extends TransportConfig> {
/**
* @return
*/
T toRealConfig(String config);
}

44
tunnel-spi/tunnel-es/pom.xml Normal file
View File

@ -0,0 +1,44 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>tunnel-spi</artifactId>
<groupId>com.hellobike.base.tunnel</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>tunnel-es</artifactId>
<dependencies>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-api</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>${es.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>${es.version}</version>
<exclusions>
<exclusion>
<artifactId>jackson-core</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-dbutils</groupId>
<artifactId>commons-dbutils</artifactId>
<version>1.7</version>
</dependency>
</dependencies>
</project>

123
tunnel-spi/tunnel-hbase/pom.xml Normal file
View File

@ -0,0 +1,123 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>tunnel-spi</artifactId>
<groupId>com.hellobike.base.tunnel</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>tunnel-hbase</artifactId>
<dependencies>
<dependency>
<groupId>com.hellobike.base.tunnel</groupId>
<artifactId>tunnel-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<artifactId>jdk.tools</artifactId>
<groupId>*</groupId>
</exclusion>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>commons-logging</artifactId>
<groupId>commons-logging</groupId>
</exclusion>
<exclusion>
<artifactId>httpclient</artifactId>
<groupId>org.apache.httpcomponents</groupId>
</exclusion>
<exclusion>
<artifactId>commons-configuration</artifactId>
<groupId>commons-configuration</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop-compat</artifactId>
<version>${hbase.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop2-compat</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
<exclusion>
<artifactId>disruptor</artifactId>
<groupId>com.lmax</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-prefix-tree</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-procedure</artifactId>
<version>${hbase.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
<version>2.4</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
</dependency>
</dependencies>
</project>

Some files were not shown because too many files have changed in this diff Show More