dromara/dbswitch
1
0
Fork 0
mirror of https://gitee.com/dromara/dbswitch.git synced 2025-09-10 05:59:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Bug: issues-I4AU42

Browse Source
This commit is contained in:
inrgihc
2021-09-20 12:46:36 +08:00
parent 057b655c7b
commit 12255c7c4b
19 changed files with 339 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@@ -387,7 +387,19 @@ bin/startup.sh
![admin_06.png](images/admin_06.png) ![admin_06.png](images/admin_06.png)
## 四、文档博客 ## 四、常见问题解决
### 1、执行启动脚本报错
- （1）在Windows下，执行startup.cmd或datasync.cmd脚本报错
> 解决办法：将脚本的内容复制后，删除原脚本文件，然后再创建一个同名脚本文件后，将原内容粘贴进去后，再执行脚本。
- （2）在Linux下，执行startup.sh或datasync.sh脚本报错
> 解决办法：用vi/vim工具打开脚本，然后命令模式下设置 ```set ff=unix```后，然后保存退出，再执行脚本。
## 五、文档博客
（1）https://blog.csdn.net/inrgihc/article/details/103739629 （1）https://blog.csdn.net/inrgihc/article/details/103739629
@@ -397,7 +409,7 @@ bin/startup.sh
（4）https://blog.csdn.net/inrgihc/article/details/103738656 （4）https://blog.csdn.net/inrgihc/article/details/103738656
## 五、问题反馈 ## 六、问题反馈
如果您看到并使用了本工具，或您觉得本工具对您有价值，请为此项目**点个赞**，以表示对本项目的支持，多谢！如果您在使用时遇到了bug，欢迎在issue中反馈。也可扫描下方二维码入群讨论：（加好友请注明："程序交流"） 如果您看到并使用了本工具，或您觉得本工具对您有价值，请为此项目**点个赞**，以表示对本项目的支持，多谢！如果您在使用时遇到了bug，欢迎在issue中反馈。也可扫描下方二维码入群讨论：（加好友请注明："程序交流"）

3
dbswitch-admin/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
</parent> </parent>
<artifactId>dbswitch-admin</artifactId> <artifactId>dbswitch-admin</artifactId>
@@ -54,6 +54,7 @@
<dependency> <dependency>
<groupId>mysql</groupId> <groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId> <artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
<scope>runtime</scope> <scope>runtime</scope>
</dependency> </dependency>

20
dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/JobExecutorService.java
View File

@@ -1,4 +1,3 @@
package com.gitee.dbswitch.admin.service;
// Copyright tang. All rights reserved. // Copyright tang. All rights reserved.
// https://gitee.com/inrgihc/dbswitch // https://gitee.com/inrgihc/dbswitch
// //
@@ -8,6 +7,8 @@ package com.gitee.dbswitch.admin.service;
// Date : 2020/1/2 // Date : 2020/1/2
// Location: beijing , china // Location: beijing , china
///////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.service;
import cn.hutool.core.exceptions.ExceptionUtil; import cn.hutool.core.exceptions.ExceptionUtil;
import com.gitee.dbswitch.admin.dao.AssignmentConfigDAO; import com.gitee.dbswitch.admin.dao.AssignmentConfigDAO;
import com.gitee.dbswitch.admin.dao.AssignmentJobDAO; import com.gitee.dbswitch.admin.dao.AssignmentJobDAO;
@@ -108,14 +109,21 @@ public class JobExecutorService extends QuartzJobBean implements InterruptableJo
try { try {
DbswichProperties properties = JsonUtil DbswichProperties properties = JsonUtil
.toBeanObject(task.getContent(), DbswichProperties.class); .toBeanObject(task.getContent(), DbswichProperties.class);
if (!assignmentConfigEntity.getFirstFlag()) {
properties.getTarget().setTargetDrop(false);
properties.getTarget().setChangeDataSynch(true);
}
MainService mainService = new MainService(properties); MainService mainService = new MainService(properties);
mainService.run(); mainService.run();
AssignmentConfigEntity config = new AssignmentConfigEntity(); if (assignmentConfigEntity.getFirstFlag()) {
config.setId(assignmentConfigEntity.getId()); AssignmentConfigEntity config = new AssignmentConfigEntity();
config.setTargetDropTable(Boolean.FALSE); config.setId(assignmentConfigEntity.getId());
config.setFirstFlag(Boolean.FALSE); config.setTargetDropTable(Boolean.FALSE);
assignmentConfigDAO.updateSelective(config); config.setFirstFlag(Boolean.FALSE);
assignmentConfigDAO.updateSelective(config);
}
assignmentJobEntity.setStatus(JobStatusEnum.PASS.getValue()); assignmentJobEntity.setStatus(JobStatusEnum.PASS.getValue());
log.info("Execute Assignment Success [taskId={}],Task Name: {}", task.getId(), log.info("Execute Assignment Success [taskId={}],Task Name: {}", task.getId(),

2
dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/type/SupportDbTypeEnum.java
View File

@@ -17,7 +17,7 @@ import org.springframework.util.StringUtils;
@AllArgsConstructor @AllArgsConstructor
public enum SupportDbTypeEnum { public enum SupportDbTypeEnum {
MYSQL(1, "mysql", "com.mysql.cj.jdbc.Driver", "/* ping */ SELECT 1", MYSQL(1, "mysql", "com.mysql.jdbc.Driver", "/* ping */ SELECT 1",
"jdbc:mysql://{host}:{port>/{name}?useUnicode=true&characterEncoding=utf-8&useSSL=false&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true&tinyInt1isBit=false"), "jdbc:mysql://{host}:{port>/{name}?useUnicode=true&characterEncoding=utf-8&useSSL=false&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true&tinyInt1isBit=false"),
MARIADB(2, "mariadb", "org.mariadb.jdbc.Driver", "SELECT 1", MARIADB(2, "mariadb", "org.mariadb.jdbc.Driver", "SELECT 1",
"jdbc:mariadb://{host}:{port}/{name}?useUnicode=true&characterEncoding=utf-8&useSSL=false&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true&tinyInt1isBit=false"), "jdbc:mariadb://{host}:{port}/{name}?useUnicode=true&characterEncoding=utf-8&useSSL=false&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true&tinyInt1isBit=false"),

2
dbswitch-common/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
</parent> </parent>
<artifactId>dbswitch-common</artifactId> <artifactId>dbswitch-common</artifactId>

9
dbswitch-core/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
</parent> </parent>
<artifactId>dbswitch-core</artifactId> <artifactId>dbswitch-core</artifactId>
@@ -21,16 +21,19 @@
<dependency> <dependency>
<groupId>mysql</groupId> <groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId> <artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.postgresql</groupId> <groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId> <artifactId>postgresql</artifactId>
<scope>runtime</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.oracle.ojdbc</groupId> <groupId>com.oracle.ojdbc</groupId>
<artifactId>ojdbc8</artifactId> <artifactId>ojdbc8</artifactId>
<scope>runtime</scope>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/com.oracle.database.nls/orai18n --> <!-- https://mvnrepository.com/artifact/com.oracle.database.nls/orai18n -->
@@ -38,12 +41,14 @@
<groupId>com.oracle.database.nls</groupId> <groupId>com.oracle.database.nls</groupId>
<artifactId>orai18n</artifactId> <artifactId>orai18n</artifactId>
<version>19.7.0.0</version> <version>19.7.0.0</version>
<scope>runtime</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.microsoft.sqlserver</groupId> <groupId>com.microsoft.sqlserver</groupId>
<artifactId>sqljdbc4</artifactId> <artifactId>sqljdbc4</artifactId>
<version>4.0</version> <version>4.0</version>
<scope>runtime</scope>
</dependency> </dependency>
<dependency> <dependency>
@@ -97,12 +102,14 @@
<dependency> <dependency>
<groupId>org.mariadb.jdbc</groupId> <groupId>org.mariadb.jdbc</groupId>
<artifactId>mariadb-java-client</artifactId> <artifactId>mariadb-java-client</artifactId>
<scope>runtime</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.ibm.db2.jcc</groupId> <groupId>com.ibm.db2.jcc</groupId>
<artifactId>db2jcc</artifactId> <artifactId>db2jcc</artifactId>
<version>db2jcc4</version> <version>db2jcc4</version>
<scope>runtime</scope>
</dependency> </dependency>
<dependency> <dependency>

2
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseMysqlImpl.java
View File

@@ -36,7 +36,7 @@ import org.springframework.util.CollectionUtils;
public class DatabaseMysqlImpl extends AbstractDatabase implements IDatabaseInterface { public class DatabaseMysqlImpl extends AbstractDatabase implements IDatabaseInterface {
public DatabaseMysqlImpl() { public DatabaseMysqlImpl() {
super("com.mysql.cj.jdbc.Driver"); super("com.mysql.jdbc.Driver");
} }
public DatabaseMysqlImpl(String driverClassName) { public DatabaseMysqlImpl(String driverClassName) {

24
dbswitch-data/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
</parent> </parent>
<artifactId>dbswitch-data</artifactId> <artifactId>dbswitch-data</artifactId>
@@ -22,6 +22,18 @@
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-core</artifactId> <artifactId>dbswitch-core</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
</dependency> </dependency>
<dependency> <dependency>
@@ -52,16 +64,6 @@
<artifactId>jackson-databind</artifactId> <artifactId>jackson-databind</artifactId>
</dependency> </dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.ehcache</groupId> <groupId>org.ehcache</groupId>
<artifactId>sizeof</artifactId> <artifactId>sizeof</artifactId>

35
dbswitch-data/src/main/java/com/gitee/dbswitch/data/DataSyncApplication.java
View File

@@ -11,6 +11,7 @@ package com.gitee.dbswitch.data;
import com.gitee.dbswitch.data.config.DbswichProperties; import com.gitee.dbswitch.data.config.DbswichProperties;
import com.gitee.dbswitch.data.core.MainService; import com.gitee.dbswitch.data.core.MainService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.Banner; import org.springframework.boot.Banner;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.WebApplicationType; import org.springframework.boot.WebApplicationType;
@@ -20,28 +21,28 @@ import org.springframework.context.ConfigurableApplicationContext;
/** /**
* DATA模块启动类 * DATA模块启动类
*
* @author tang
* *
* @author tang
*/ */
@Slf4j
@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class}) @SpringBootApplication(exclude = {DataSourceAutoConfiguration.class})
public class DataSyncApplication { public class DataSyncApplication {
public static void main(String[] args) { public static void main(String[] args) {
SpringApplication springApplication = new SpringApplication(DataSyncApplication.class); SpringApplication springApplication = new SpringApplication(DataSyncApplication.class);
springApplication.setWebApplicationType(WebApplicationType.NONE); springApplication.setWebApplicationType(WebApplicationType.NONE);
springApplication.setBannerMode(Banner.Mode.OFF); springApplication.setBannerMode(Banner.Mode.OFF);
ConfigurableApplicationContext applicationContext = springApplication.run(args); ConfigurableApplicationContext applicationContext = springApplication.run(args);
try { try {
DbswichProperties properties = applicationContext.getBean(DbswichProperties.class); DbswichProperties properties = applicationContext.getBean(DbswichProperties.class);
MainService mainService = new MainService(properties); MainService mainService = new MainService(properties);
mainService.run(); mainService.run();
}catch (Exception e){ } catch (Exception e) {
log.error("error:", e);
} finally {
applicationContext.close();
}
} finally { }
applicationContext.close();
}
}
} }

352
dbswitch-data/src/main/java/com/gitee/dbswitch/data/core/MainService.java
View File

@@ -9,6 +9,18 @@
///////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////
package com.gitee.dbswitch.data.core; package com.gitee.dbswitch.data.core;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.gitee.dbswitch.core.model.TableDescription;
import com.gitee.dbswitch.core.service.IMetaDataService;
import com.gitee.dbswitch.core.service.impl.MigrationMetaDataServiceImpl;
import com.gitee.dbswitch.data.config.DbswichProperties;
import com.gitee.dbswitch.data.domain.PerfStat;
import com.gitee.dbswitch.data.handler.MigrationHandler;
import com.gitee.dbswitch.data.util.BytesUnitUtils;
import com.gitee.dbswitch.data.util.DataSouceUtils;
import com.gitee.dbswitch.data.util.JdbcTemplateUtils;
import com.gitee.dbswitch.data.util.StrUtils;
import com.zaxxer.hikari.HikariDataSource;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
@@ -17,20 +29,8 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.Supplier; import java.util.function.Supplier;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import com.gitee.dbswitch.data.domain.PerfStat;
import com.gitee.dbswitch.data.handler.MigrationHandler;
import com.gitee.dbswitch.data.util.BytesUnitUtils;
import com.gitee.dbswitch.data.util.DataSouceUtils;
import com.gitee.dbswitch.data.util.StrUtils;
import com.gitee.dbswitch.core.service.impl.MigrationMetaDataServiceImpl;
import org.springframework.util.StopWatch;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.gitee.dbswitch.core.model.TableDescription;
import com.gitee.dbswitch.core.service.IMetaDataService;
import com.gitee.dbswitch.data.config.DbswichProperties;
import com.gitee.dbswitch.data.util.JdbcTemplateUtils;
import com.zaxxer.hikari.HikariDataSource;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.util.StopWatch;
/** /**
* 数据迁移主逻辑类 * 数据迁移主逻辑类
@@ -40,183 +40,195 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
public class MainService { public class MainService {
/** /**
* JSON序列化工具 * JSON序列化工具
*/ */
private final ObjectMapper jackson = new ObjectMapper(); private final ObjectMapper jackson = new ObjectMapper();
/** /**
* 性能统计记录表 * 性能统计记录表
*/ */
private final List<PerfStat> perfStats = new ArrayList<>(); private final List<PerfStat> perfStats = new ArrayList<>();
/** /**
* 配置参数 * 配置参数
*/ */
private DbswichProperties properties; private final DbswichProperties properties;
/** /**
* 构造函数 * 构造函数
* *
* @param properties 配置信息 * @param properties 配置信息
*/ */
public MainService(DbswichProperties properties){ public MainService(DbswichProperties properties) {
this.properties=properties; this.properties = properties;
} }
/** /**
* 执行主逻辑 * 执行主逻辑
*/ */
public void run() throws Exception { public void run() throws Exception {
StopWatch watch = new StopWatch(); StopWatch watch = new StopWatch();
watch.start(); watch.start();
log.info("dbswitch data service is started...."); log.info("dbswitch data service is started....");
//log.info("Application properties configuration \n{}", properties);
try { try (HikariDataSource targetDataSource = DataSouceUtils
//log.info("Application properties configuration \n{}", properties); .createTargetDataSource(properties.getTarget())) {
List<DbswichProperties.SourceDataSourceProperties> sourcesProperties = properties.getSource(); int sourcePropertiesIndex = 0;
int totalTableCount = 0;
List<DbswichProperties.SourceDataSourceProperties> sourcesProperties = properties.getSource();
for (DbswichProperties.SourceDataSourceProperties sourceProperties : sourcesProperties) {
HikariDataSource targetDataSource = DataSouceUtils.createTargetDataSource(properties.getTarget()); try (HikariDataSource sourceDataSource = DataSouceUtils
.createSourceDataSource(sourceProperties)) {
IMetaDataService sourceMetaDataService = new MigrationMetaDataServiceImpl();
int sourcePropertiesIndex = 0; sourceMetaDataService
int totalTableCount = 0; .setDatabaseConnection(JdbcTemplateUtils.getDatabaseProduceName(sourceDataSource));
for (DbswichProperties.SourceDataSourceProperties sourceProperties : sourcesProperties) {
try (HikariDataSource sourceDataSource = DataSouceUtils.createSourceDataSource(sourceProperties)) { // 判断处理的策略：是排除还是包含
IMetaDataService sourceMetaDataService = new MigrationMetaDataServiceImpl(); List<String> includes = StrUtils.stringToList(sourceProperties.getSourceIncludes());
log.info("Includes tables is :{}", jackson.writeValueAsString(includes));
List<String> filters = StrUtils.stringToList(sourceProperties.getSourceExcludes());
log.info("Filter tables is :{}", jackson.writeValueAsString(filters));
sourceMetaDataService.setDatabaseConnection(JdbcTemplateUtils.getDatabaseProduceName(sourceDataSource)); boolean useExcludeTables = includes.isEmpty();
if (useExcludeTables) {
log.info("!!!! Use dbswitch.source[{}].source-excludes parameter to filter tables",
sourcePropertiesIndex);
} else {
log.info("!!!! Use dbswitch.source[{}].source-includes parameter to filter tables",
sourcePropertiesIndex);
}
// 判断处理的策略：是排除还是包含 List<CompletableFuture<Void>> futures = new ArrayList<>();
List<String> includes = StrUtils.stringToList(sourceProperties.getSourceIncludes());
log.info("Includes tables is :{}", jackson.writeValueAsString(includes));
List<String> filters = StrUtils.stringToList(sourceProperties.getSourceExcludes());
log.info("Filter tables is :{}", jackson.writeValueAsString(filters));
boolean useExcludeTables = includes.isEmpty(); List<String> schemas = StrUtils.stringToList(sourceProperties.getSourceSchema());
if (useExcludeTables) { log.info("Source schema names is :{}", jackson.writeValueAsString(schemas));
log.info("!!!! Use dbswitch.source[{}].source-excludes parameter to filter tables", sourcePropertiesIndex);
} else { AtomicInteger numberOfFailures = new AtomicInteger(0);
log.info("!!!! Use dbswitch.source[{}].source-includes parameter to filter tables", sourcePropertiesIndex); AtomicLong totalBytesSize = new AtomicLong(0L);
final int indexInternal = sourcePropertiesIndex;
for (String schema : schemas) {
List<TableDescription> tableList = sourceMetaDataService
.queryTableList(sourceProperties.getUrl(),
sourceProperties.getUsername(), sourceProperties.getPassword(), schema);
if (tableList.isEmpty()) {
log.warn("### Find source database table list empty for schema name is : {}", schema);
} else {
for (TableDescription td : tableList) {
String tableName = td.getTableName();
if (useExcludeTables) {
if (!filters.contains(tableName)) {
futures.add(
makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource,
numberOfFailures, totalBytesSize));
}
} else {
if (includes.size() == 1 && (includes.get(0).contains("*") || includes.get(0)
.contains("?"))) {
if (Pattern.matches(includes.get(0), tableName)) {
futures.add(
makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource,
numberOfFailures, totalBytesSize));
} }
} else if (includes.contains(tableName)) {
List<CompletableFuture<Void>> futures = new ArrayList<>(); futures.add(
makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource,
List<String> schemas = StrUtils.stringToList(sourceProperties.getSourceSchema()); numberOfFailures, totalBytesSize));
log.info("Source schema names is :{}", jackson.writeValueAsString(schemas)); }
AtomicInteger numberOfFailures = new AtomicInteger(0);
AtomicLong totalBytesSize = new AtomicLong(0L);
final int indexInternal = sourcePropertiesIndex;
for (String schema : schemas) {
// 读取源库指定schema里所有的表
List<TableDescription> tableList = sourceMetaDataService.queryTableList(sourceProperties.getUrl(),
sourceProperties.getUsername(), sourceProperties.getPassword(), schema);
if (tableList.isEmpty()) {
log.warn("### Find source database table list empty for schema name is : {}", schema);
} else {
for (TableDescription td : tableList) {
String tableName = td.getTableName();
if (useExcludeTables) {
if (!filters.contains(tableName)) {
futures.add(makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource, numberOfFailures, totalBytesSize));
}
} else {
if (includes.size() == 1 && (includes.get(0).contains("*") || includes.get(0).contains("?"))) {
if (Pattern.matches(includes.get(0), tableName)) {
futures.add(makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource, numberOfFailures, totalBytesSize));
}
} else if (includes.contains(tableName)) {
futures.add(makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource, numberOfFailures, totalBytesSize));
}
}
}
}
}
CompletableFuture.allOf(futures.toArray(new CompletableFuture[]{})).join();
log.info("#### Complete data migration for the [ {} ] data source:\ntotal count={}\nfailure count={}\ntotal bytes size={}",
sourcePropertiesIndex, futures.size(), numberOfFailures.get(), BytesUnitUtils.bytesSizeToHuman(totalBytesSize.get()));
perfStats.add(new PerfStat(sourcePropertiesIndex, futures.size(), numberOfFailures.get(), totalBytesSize.get()));
++sourcePropertiesIndex;
totalTableCount += futures.size();
} }
}
} }
log.info("service run all success, total migrate table count={} ", totalTableCount);
} catch (Exception e) {
log.error("error:", e);
throw e;
} finally {
watch.stop();
log.info("total elipse = {} s", watch.getTotalTimeSeconds());
StringBuilder sb = new StringBuilder(); }
sb.append("===================================\n");
sb.append(String.format("total elipse time:\t %f s\n", watch.getTotalTimeSeconds())); CompletableFuture.allOf(futures.toArray(new CompletableFuture[]{})).join();
sb.append("-------------------------------------\n"); log.info(
perfStats.forEach(st -> { "#### Complete data migration for the [ {} ] data source:\ntotal count={}\nfailure count={}\ntotal bytes size={}",
sb.append(st); sourcePropertiesIndex, futures.size(), numberOfFailures.get(),
if(perfStats.size()>1){ BytesUnitUtils.bytesSizeToHuman(totalBytesSize.get()));
sb.append("-------------------------------------\n");
} perfStats.add(new PerfStat(sourcePropertiesIndex, futures.size(), numberOfFailures.get(),
}); totalBytesSize.get()));
sb.append("===================================\n");
log.info("\n\n" + sb.toString()); ++sourcePropertiesIndex;
totalTableCount += futures.size();
} }
} }
log.info("service run all success, total migrate table count={} ", totalTableCount);
} finally {
watch.stop();
log.info("total ellipse = {} s", watch.getTotalTimeSeconds());
/** StringBuilder sb = new StringBuilder();
* 构造一个异步执行任务 sb.append("===================================\n");
* sb.append(String.format("total ellipse time:\t %f s\n", watch.getTotalTimeSeconds()));
* @param td 表描述上下文 sb.append("-------------------------------------\n");
* @param indexInternal 源端索引号 perfStats.forEach(st -> {
* @param sds 源端的DataSource数据源 sb.append(st);
* @param tds 目的端的DataSource数据源 if (perfStats.size() > 1) {
* @param numberOfFailures 失败的数量 sb.append("-------------------------------------\n");
* @param totalBytesSize 同步的字节大小 }
* @return CompletableFuture<Void> });
*/ sb.append("===================================\n");
private CompletableFuture<Void> makeFutureTask(TableDescription td, Integer indexInternal, log.info("\n\n" + sb.toString());
HikariDataSource sds, HikariDataSource tds,
AtomicInteger numberOfFailures, AtomicLong totalBytesSize) {
return CompletableFuture.supplyAsync(getMigrateHandler(td, indexInternal, sds, tds))
.exceptionally(getExceptHandler(td, numberOfFailures))
.thenAccept(totalBytesSize::addAndGet);
} }
}
/** /**
* 单表迁移处理方法 * 构造一个异步执行任务
* *
* @param td 表描述上下文 * @param td 表描述上下文
* @param indexInternal 源端索引号 * @param indexInternal 源端索引号
* @param sds 源端的DataSource数据源 * @param sds 源端的DataSource数据源
* @param tds 目的端的DataSource数据源 * @param tds 目的端的DataSource数据源
* @return Supplier<Long> * @param numberOfFailures 失败的数量
*/ * @param totalBytesSize 同步的字节大小
private Supplier<Long> getMigrateHandler(TableDescription td, Integer indexInternal, HikariDataSource sds, HikariDataSource tds) { * @return CompletableFuture<Void>
return () -> MigrationHandler.createInstance(td, properties, indexInternal, sds, tds).get(); */
} private CompletableFuture<Void> makeFutureTask(TableDescription td, Integer indexInternal,
HikariDataSource sds, HikariDataSource tds,
AtomicInteger numberOfFailures, AtomicLong totalBytesSize) {
return CompletableFuture.supplyAsync(getMigrateHandler(td, indexInternal, sds, tds))
.exceptionally(getExceptHandler(td, numberOfFailures))
.thenAccept(totalBytesSize::addAndGet);
}
/** /**
* 异常处理函数方法 * 单表迁移处理方法
* *
* @param td 表描述上下文 * @param td 表描述上下文
* @param numberOfFailures 失败记录数 * @param indexInternal 源端索引号
* @return Function<Throwable, Long> * @param sds 源端的DataSource数据源
*/ * @param tds 目的端的DataSource数据源
private Function<Throwable, Long> getExceptHandler(TableDescription td, AtomicInteger numberOfFailures) { * @return Supplier<Long>
return (e) -> { */
log.error("Error migration for table: {}.{}, error message:", td.getSchemaName(), td.getTableName(), e); private Supplier<Long> getMigrateHandler(TableDescription td, Integer indexInternal,
numberOfFailures.incrementAndGet(); HikariDataSource sds, HikariDataSource tds) {
throw new RuntimeException(e); return () -> MigrationHandler.createInstance(td, properties, indexInternal, sds, tds).get();
}; }
}
/**
* 异常处理函数方法
*
* @param td 表描述上下文
* @param numberOfFailures 失败记录数
* @return Function<Throwable, Long>
*/
private Function<Throwable, Long> getExceptHandler(TableDescription td,
AtomicInteger numberOfFailures) {
return (e) -> {
log.error("Error migration for table: {}.{}, error message:", td.getSchemaName(),
td.getTableName(), e);
numberOfFailures.incrementAndGet();
throw new RuntimeException(e);
};
}
} }

2
dbswitch-dbchange/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
</parent> </parent>
<artifactId>dbswitch-dbchange</artifactId> <artifactId>dbswitch-dbchange</artifactId>

2
dbswitch-dbcommon/pom.xml
View File

@@ -3,7 +3,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
</parent> </parent>
<artifactId>dbswitch-dbcommon</artifactId> <artifactId>dbswitch-dbcommon</artifactId>

2
dbswitch-dbsynch/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
</parent> </parent>
<artifactId>dbswitch-dbsynch</artifactId> <artifactId>dbswitch-dbsynch</artifactId>

2
dbswitch-dbwriter/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
</parent> </parent>
<artifactId>dbswitch-dbwriter</artifactId> <artifactId>dbswitch-dbwriter</artifactId>

2
dbswitch-pgwriter/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
</parent> </parent>
<artifactId>dbswitch-pgwriter</artifactId> <artifactId>dbswitch-pgwriter</artifactId>

2
dbswitch-sql/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
</parent> </parent>
<artifactId>dbswitch-sql</artifactId> <artifactId>dbswitch-sql</artifactId>

2
package-tool/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
</parent> </parent>
<artifactId>package-tool</artifactId> <artifactId>package-tool</artifactId>

154
pom.xml
View File

@@ -1,85 +1,85 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" <project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent> <parent>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId> <artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.0.RELEASE</version> <version>2.2.0.RELEASE</version>
<relativePath /> <!-- lookup parent from repository --> <relativePath/> <!-- lookup parent from repository -->
</parent> </parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version> <version>1.6.1</version>
<packaging>pom</packaging> <packaging>pom</packaging>
<name>dbswitch</name> <name>dbswitch</name>
<description>database switch project</description> <description>database switch project</description>
<properties> <properties>
<java.version>1.8</java.version> <java.version>1.8</java.version>
<maven-jar-plugin.version>3.1.1</maven-jar-plugin.version> <maven-jar-plugin.version>3.1.1</maven-jar-plugin.version>
<maven.test.skip>true</maven.test.skip> <maven.test.skip>true</maven.test.skip>
<maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties> </properties>
<modules> <modules>
<module>dbswitch-common</module> <module>dbswitch-common</module>
<module>dbswitch-core</module> <module>dbswitch-core</module>
<module>dbswitch-pgwriter</module> <module>dbswitch-pgwriter</module>
<module>dbswitch-dbwriter</module> <module>dbswitch-dbwriter</module>
<module>dbswitch-dbcommon</module> <module>dbswitch-dbcommon</module>
<module>dbswitch-dbchange</module> <module>dbswitch-dbchange</module>
<module>dbswitch-dbsynch</module> <module>dbswitch-dbsynch</module>
<module>dbswitch-data</module> <module>dbswitch-data</module>
<module>dbswitch-sql</module> <module>dbswitch-sql</module>
<module>dbswitch-admin</module> <module>dbswitch-admin</module>
<module>package-tool</module> <module>package-tool</module>
</modules> </modules>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId> <artifactId>spring-boot-starter</artifactId>
</dependency> </dependency>
</dependencies> </dependencies>
<build> <build>
<plugins> <plugins>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId> <artifactId>maven-jar-plugin</artifactId>
<configuration> <configuration>
<excludes> <excludes>
<exclude>**/*.xml</exclude> <exclude>**/*.xml</exclude>
<exclude>**/*.yml</exclude> <exclude>**/*.yml</exclude>
<exclude>**/*.properties</exclude> <exclude>**/*.properties</exclude>
</excludes> </excludes>
</configuration> </configuration>
</plugin> </plugin>
<!-- 将依赖的jar包拷贝到target目录下 --> <!-- 将依赖的jar包拷贝到target目录下 -->
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId> <artifactId>maven-dependency-plugin</artifactId>
<executions> <executions>
<execution> <execution>
<id>copy-dependencies</id> <id>copy-dependencies</id>
<phase>package</phase> <phase>package</phase>
<goals> <goals>
<goal>copy-dependencies</goal> <goal>copy-dependencies</goal>
</goals> </goals>
<configuration> <configuration>
<outputDirectory>${basedir}/target</outputDirectory> <outputDirectory>${basedir}/target</outputDirectory>
<excludeTransitive>false</excludeTransitive> <excludeTransitive>false</excludeTransitive>
<stripVersion>false</stripVersion> <stripVersion>false</stripVersion>
</configuration> </configuration>
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
</plugins> </plugins>
</build> </build>
</project> </project>

2
version.cmd
View File

@@ -1,6 +1,6 @@
@echo off @echo off
set APP_VERSION=1.6.0 set APP_VERSION=1.6.1
echo "Clean Project ..." echo "Clean Project ..."
call mvn clean -f pom.xml call mvn clean -f pom.xml