dromara/dbswitch
1
0
Fork 0
mirror of https://gitee.com/dromara/dbswitch.git synced 2025-09-10 05:59:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Bug: issues-I4AU42

Browse Source
This commit is contained in:
inrgihc
2021-09-20 12:46:36 +08:00
parent 057b655c7b
commit 12255c7c4b
19 changed files with 339 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@@ -387,7 +387,19 @@ bin/startup.sh
![admin_06.png](images/admin_06.png)
## 四、文档博客
## 四、常见问题解决
### 1、执行启动脚本报错
- （1）在Windows下，执行startup.cmd或datasync.cmd脚本报错
> 解决办法：将脚本的内容复制后，删除原脚本文件，然后再创建一个同名脚本文件后，将原内容粘贴进去后，再执行脚本。
- （2）在Linux下，执行startup.sh或datasync.sh脚本报错
> 解决办法：用vi/vim工具打开脚本，然后命令模式下设置 ```set ff=unix```后，然后保存退出，再执行脚本。
## 五、文档博客
（1）https://blog.csdn.net/inrgihc/article/details/103739629
@@ -397,7 +409,7 @@ bin/startup.sh
（4）https://blog.csdn.net/inrgihc/article/details/103738656
## 五、问题反馈
## 六、问题反馈
如果您看到并使用了本工具，或您觉得本工具对您有价值，请为此项目**点个赞**，以表示对本项目的支持，多谢！如果您在使用时遇到了bug，欢迎在issue中反馈。也可扫描下方二维码入群讨论：（加好友请注明："程序交流"）

3
dbswitch-admin/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
</parent>
<artifactId>dbswitch-admin</artifactId>
@@ -54,6 +54,7 @@
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
<scope>runtime</scope>
</dependency>

10
dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/JobExecutorService.java
View File

@@ -1,4 +1,3 @@
package com.gitee.dbswitch.admin.service;
// Copyright tang. All rights reserved.
// https://gitee.com/inrgihc/dbswitch
//
@@ -8,6 +7,8 @@ package com.gitee.dbswitch.admin.service;
// Date : 2020/1/2
// Location: beijing , china
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.service;
import cn.hutool.core.exceptions.ExceptionUtil;
import com.gitee.dbswitch.admin.dao.AssignmentConfigDAO;
import com.gitee.dbswitch.admin.dao.AssignmentJobDAO;
@@ -108,14 +109,21 @@ public class JobExecutorService extends QuartzJobBean implements InterruptableJo
try {
DbswichProperties properties = JsonUtil
.toBeanObject(task.getContent(), DbswichProperties.class);
if (!assignmentConfigEntity.getFirstFlag()) {
properties.getTarget().setTargetDrop(false);
properties.getTarget().setChangeDataSynch(true);
}
MainService mainService = new MainService(properties);
mainService.run();
if (assignmentConfigEntity.getFirstFlag()) {
AssignmentConfigEntity config = new AssignmentConfigEntity();
config.setId(assignmentConfigEntity.getId());
config.setTargetDropTable(Boolean.FALSE);
config.setFirstFlag(Boolean.FALSE);
assignmentConfigDAO.updateSelective(config);
}
assignmentJobEntity.setStatus(JobStatusEnum.PASS.getValue());
log.info("Execute Assignment Success [taskId={}],Task Name: {}", task.getId(),

2
dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/type/SupportDbTypeEnum.java
View File

@@ -17,7 +17,7 @@ import org.springframework.util.StringUtils;
@AllArgsConstructor
public enum SupportDbTypeEnum {
MYSQL(1, "mysql", "com.mysql.cj.jdbc.Driver", "/* ping */ SELECT 1",
MYSQL(1, "mysql", "com.mysql.jdbc.Driver", "/* ping */ SELECT 1",
"jdbc:mysql://{host}:{port>/{name}?useUnicode=true&characterEncoding=utf-8&useSSL=false&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true&tinyInt1isBit=false"),
MARIADB(2, "mariadb", "org.mariadb.jdbc.Driver", "SELECT 1",
"jdbc:mariadb://{host}:{port}/{name}?useUnicode=true&characterEncoding=utf-8&useSSL=false&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai&nullCatalogMeansCurrent=true&tinyInt1isBit=false"),

2
dbswitch-common/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
</parent>
<artifactId>dbswitch-common</artifactId>

9
dbswitch-core/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
</parent>
<artifactId>dbswitch-core</artifactId>
@@ -21,16 +21,19 @@
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.oracle.ojdbc</groupId>
<artifactId>ojdbc8</artifactId>
<scope>runtime</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/com.oracle.database.nls/orai18n -->
@@ -38,12 +41,14 @@
<groupId>com.oracle.database.nls</groupId>
<artifactId>orai18n</artifactId>
<version>19.7.0.0</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>sqljdbc4</artifactId>
<version>4.0</version>
<scope>runtime</scope>
</dependency>
<dependency>
@@ -97,12 +102,14 @@
<dependency>
<groupId>org.mariadb.jdbc</groupId>
<artifactId>mariadb-java-client</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.ibm.db2.jcc</groupId>
<artifactId>db2jcc</artifactId>
<version>db2jcc4</version>
<scope>runtime</scope>
</dependency>
<dependency>

2
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseMysqlImpl.java
View File

@@ -36,7 +36,7 @@ import org.springframework.util.CollectionUtils;
public class DatabaseMysqlImpl extends AbstractDatabase implements IDatabaseInterface {
public DatabaseMysqlImpl() {
super("com.mysql.cj.jdbc.Driver");
super("com.mysql.jdbc.Driver");
}
public DatabaseMysqlImpl(String driverClassName) {

24
dbswitch-data/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
</parent>
<artifactId>dbswitch-data</artifactId>
@@ -22,6 +22,18 @@
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-core</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
</dependency>
<dependency>
@@ -52,16 +64,6 @@
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
</dependency>
<dependency>
<groupId>org.ehcache</groupId>
<artifactId>sizeof</artifactId>

7
dbswitch-data/src/main/java/com/gitee/dbswitch/data/DataSyncApplication.java
View File

@@ -11,6 +11,7 @@ package com.gitee.dbswitch.data;
import com.gitee.dbswitch.data.config.DbswichProperties;
import com.gitee.dbswitch.data.core.MainService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.Banner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.WebApplicationType;
@@ -22,8 +23,8 @@ import org.springframework.context.ConfigurableApplicationContext;
* DATA模块启动类
*
* @author tang
*
*/
@Slf4j
@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class})
public class DataSyncApplication {
@@ -36,8 +37,8 @@ public class DataSyncApplication {
DbswichProperties properties = applicationContext.getBean(DbswichProperties.class);
MainService mainService = new MainService(properties);
mainService.run();
}catch (Exception e){
} catch (Exception e) {
log.error("error:", e);
} finally {
applicationContext.close();
}

98
dbswitch-data/src/main/java/com/gitee/dbswitch/data/core/MainService.java
View File

@@ -9,6 +9,18 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.data.core;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.gitee.dbswitch.core.model.TableDescription;
import com.gitee.dbswitch.core.service.IMetaDataService;
import com.gitee.dbswitch.core.service.impl.MigrationMetaDataServiceImpl;
import com.gitee.dbswitch.data.config.DbswichProperties;
import com.gitee.dbswitch.data.domain.PerfStat;
import com.gitee.dbswitch.data.handler.MigrationHandler;
import com.gitee.dbswitch.data.util.BytesUnitUtils;
import com.gitee.dbswitch.data.util.DataSouceUtils;
import com.gitee.dbswitch.data.util.JdbcTemplateUtils;
import com.gitee.dbswitch.data.util.StrUtils;
import com.zaxxer.hikari.HikariDataSource;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
@@ -17,20 +29,8 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import com.gitee.dbswitch.data.domain.PerfStat;
import com.gitee.dbswitch.data.handler.MigrationHandler;
import com.gitee.dbswitch.data.util.BytesUnitUtils;
import com.gitee.dbswitch.data.util.DataSouceUtils;
import com.gitee.dbswitch.data.util.StrUtils;
import com.gitee.dbswitch.core.service.impl.MigrationMetaDataServiceImpl;
import org.springframework.util.StopWatch;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.gitee.dbswitch.core.model.TableDescription;
import com.gitee.dbswitch.core.service.IMetaDataService;
import com.gitee.dbswitch.data.config.DbswichProperties;
import com.gitee.dbswitch.data.util.JdbcTemplateUtils;
import com.zaxxer.hikari.HikariDataSource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.StopWatch;
/**
* 数据迁移主逻辑类
@@ -53,15 +53,15 @@ public class MainService {
/**
* 配置参数
*/
private DbswichProperties properties;
private final DbswichProperties properties;
/**
* 构造函数
*
* @param properties 配置信息
*/
public MainService(DbswichProperties properties){
this.properties=properties;
public MainService(DbswichProperties properties) {
this.properties = properties;
}
/**
@@ -72,21 +72,21 @@ public class MainService {
watch.start();
log.info("dbswitch data service is started....");
try {
//log.info("Application properties configuration \n{}", properties);
List<DbswichProperties.SourceDataSourceProperties> sourcesProperties = properties.getSource();
HikariDataSource targetDataSource = DataSouceUtils.createTargetDataSource(properties.getTarget());
try (HikariDataSource targetDataSource = DataSouceUtils
.createTargetDataSource(properties.getTarget())) {
int sourcePropertiesIndex = 0;
int totalTableCount = 0;
List<DbswichProperties.SourceDataSourceProperties> sourcesProperties = properties.getSource();
for (DbswichProperties.SourceDataSourceProperties sourceProperties : sourcesProperties) {
try (HikariDataSource sourceDataSource = DataSouceUtils.createSourceDataSource(sourceProperties)) {
try (HikariDataSource sourceDataSource = DataSouceUtils
.createSourceDataSource(sourceProperties)) {
IMetaDataService sourceMetaDataService = new MigrationMetaDataServiceImpl();
sourceMetaDataService.setDatabaseConnection(JdbcTemplateUtils.getDatabaseProduceName(sourceDataSource));
sourceMetaDataService
.setDatabaseConnection(JdbcTemplateUtils.getDatabaseProduceName(sourceDataSource));
// 判断处理的策略：是排除还是包含
List<String> includes = StrUtils.stringToList(sourceProperties.getSourceIncludes());
@@ -96,9 +96,11 @@ public class MainService {
boolean useExcludeTables = includes.isEmpty();
if (useExcludeTables) {
log.info("!!!! Use dbswitch.source[{}].source-excludes parameter to filter tables", sourcePropertiesIndex);
log.info("!!!! Use dbswitch.source[{}].source-excludes parameter to filter tables",
sourcePropertiesIndex);
} else {
log.info("!!!! Use dbswitch.source[{}].source-includes parameter to filter tables", sourcePropertiesIndex);
log.info("!!!! Use dbswitch.source[{}].source-includes parameter to filter tables",
sourcePropertiesIndex);
}
List<CompletableFuture<Void>> futures = new ArrayList<>();
@@ -110,8 +112,8 @@ public class MainService {
AtomicLong totalBytesSize = new AtomicLong(0L);
final int indexInternal = sourcePropertiesIndex;
for (String schema : schemas) {
// 读取源库指定schema里所有的表
List<TableDescription> tableList = sourceMetaDataService.queryTableList(sourceProperties.getUrl(),
List<TableDescription> tableList = sourceMetaDataService
.queryTableList(sourceProperties.getUrl(),
sourceProperties.getUsername(), sourceProperties.getPassword(), schema);
if (tableList.isEmpty()) {
log.warn("### Find source database table list empty for schema name is : {}", schema);
@@ -121,15 +123,22 @@ public class MainService {
if (useExcludeTables) {
if (!filters.contains(tableName)) {
futures.add(makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource, numberOfFailures, totalBytesSize));
futures.add(
makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource,
numberOfFailures, totalBytesSize));
}
} else {
if (includes.size() == 1 && (includes.get(0).contains("*") || includes.get(0).contains("?"))) {
if (includes.size() == 1 && (includes.get(0).contains("*") || includes.get(0)
.contains("?"))) {
if (Pattern.matches(includes.get(0), tableName)) {
futures.add(makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource, numberOfFailures, totalBytesSize));
futures.add(
makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource,
numberOfFailures, totalBytesSize));
}
} else if (includes.contains(tableName)) {
futures.add(makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource, numberOfFailures, totalBytesSize));
futures.add(
makeFutureTask(td, indexInternal, sourceDataSource, targetDataSource,
numberOfFailures, totalBytesSize));
}
}
@@ -140,30 +149,30 @@ public class MainService {
}
CompletableFuture.allOf(futures.toArray(new CompletableFuture[]{})).join();
log.info("#### Complete data migration for the [ {} ] data source:\ntotal count={}\nfailure count={}\ntotal bytes size={}",
sourcePropertiesIndex, futures.size(), numberOfFailures.get(), BytesUnitUtils.bytesSizeToHuman(totalBytesSize.get()));
log.info(
"#### Complete data migration for the [ {} ] data source:\ntotal count={}\nfailure count={}\ntotal bytes size={}",
sourcePropertiesIndex, futures.size(), numberOfFailures.get(),
BytesUnitUtils.bytesSizeToHuman(totalBytesSize.get()));
perfStats.add(new PerfStat(sourcePropertiesIndex, futures.size(), numberOfFailures.get(), totalBytesSize.get()));
perfStats.add(new PerfStat(sourcePropertiesIndex, futures.size(), numberOfFailures.get(),
totalBytesSize.get()));
++sourcePropertiesIndex;
totalTableCount += futures.size();
}
}
log.info("service run all success, total migrate table count={} ", totalTableCount);
} catch (Exception e) {
log.error("error:", e);
throw e;
} finally {
watch.stop();
log.info("total elipse = {} s", watch.getTotalTimeSeconds());
log.info("total ellipse = {} s", watch.getTotalTimeSeconds());
StringBuilder sb = new StringBuilder();
sb.append("===================================\n");
sb.append(String.format("total elipse time:\t %f s\n", watch.getTotalTimeSeconds()));
sb.append(String.format("total ellipse time:\t %f s\n", watch.getTotalTimeSeconds()));
sb.append("-------------------------------------\n");
perfStats.forEach(st -> {
sb.append(st);
if(perfStats.size()>1){
if (perfStats.size() > 1) {
sb.append("-------------------------------------\n");
}
});
@@ -200,7 +209,8 @@ public class MainService {
* @param tds 目的端的DataSource数据源
* @return Supplier<Long>
*/
private Supplier<Long> getMigrateHandler(TableDescription td, Integer indexInternal, HikariDataSource sds, HikariDataSource tds) {
private Supplier<Long> getMigrateHandler(TableDescription td, Integer indexInternal,
HikariDataSource sds, HikariDataSource tds) {
return () -> MigrationHandler.createInstance(td, properties, indexInternal, sds, tds).get();
}
@@ -211,9 +221,11 @@ public class MainService {
* @param numberOfFailures 失败记录数
* @return Function<Throwable, Long>
*/
private Function<Throwable, Long> getExceptHandler(TableDescription td, AtomicInteger numberOfFailures) {
private Function<Throwable, Long> getExceptHandler(TableDescription td,
AtomicInteger numberOfFailures) {
return (e) -> {
log.error("Error migration for table: {}.{}, error message:", td.getSchemaName(), td.getTableName(), e);
log.error("Error migration for table: {}.{}, error message:", td.getSchemaName(),
td.getTableName(), e);
numberOfFailures.incrementAndGet();
throw new RuntimeException(e);
};

2
dbswitch-dbchange/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
</parent>
<artifactId>dbswitch-dbchange</artifactId>

2
dbswitch-dbcommon/pom.xml
View File

@@ -3,7 +3,7 @@
<parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
</parent>
<artifactId>dbswitch-dbcommon</artifactId>

2
dbswitch-dbsynch/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
</parent>
<artifactId>dbswitch-dbsynch</artifactId>

2
dbswitch-dbwriter/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
</parent>
<artifactId>dbswitch-dbwriter</artifactId>

2
dbswitch-pgwriter/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
</parent>
<artifactId>dbswitch-pgwriter</artifactId>

2
dbswitch-sql/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
</parent>
<artifactId>dbswitch-sql</artifactId>

2
package-tool/pom.xml
View File

@@ -5,7 +5,7 @@
<parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
</parent>
<artifactId>package-tool</artifactId>

4
pom.xml
View File

@@ -6,12 +6,12 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.0.RELEASE</version>
<relativePath /> <!-- lookup parent from repository -->
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId>
<version>1.6.0</version>
<version>1.6.1</version>
<packaging>pom</packaging>
<name>dbswitch</name>
<description>database switch project</description>

2
version.cmd
View File

@@ -1,6 +1,6 @@
@echo off
set APP_VERSION=1.6.0
set APP_VERSION=1.6.1
echo "Clean Project ..."
call mvn clean -f pom.xml