dbswitch模块代码梳理调整

This commit is contained in:
inrgihc
2023-07-24 22:46:13 +08:00
parent e271a117d1
commit 208406a876
33 changed files with 215 additions and 309 deletions

View File

@@ -10,7 +10,7 @@
package com.gitee.dbswitch.admin.controller.converter;
import cn.hutool.extra.spring.SpringUtil;
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
import com.gitee.dbswitch.common.converter.AbstractConverter;
import com.gitee.dbswitch.admin.dao.AssignmentConfigDAO;
import com.gitee.dbswitch.admin.dao.DatabaseConnectionDAO;
import com.gitee.dbswitch.admin.entity.AssignmentConfigEntity;

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.controller.converter;
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
import com.gitee.dbswitch.common.converter.AbstractConverter;
import com.gitee.dbswitch.admin.entity.AssignmentTaskEntity;
import com.gitee.dbswitch.admin.model.response.AssignmentInfoResponse;

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.controller.converter;
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
import com.gitee.dbswitch.common.converter.AbstractConverter;
import com.gitee.dbswitch.admin.entity.DatabaseConnectionEntity;
import com.gitee.dbswitch.admin.model.response.DbConnectionDetailResponse;
import java.util.Objects;

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.controller.converter;
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
import com.gitee.dbswitch.common.converter.AbstractConverter;
import com.gitee.dbswitch.admin.entity.SystemLogEntity;
import com.gitee.dbswitch.admin.model.response.SystemLogDetailResponse;
import java.util.Objects;

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.controller.converter;
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
import com.gitee.dbswitch.common.converter.AbstractConverter;
import com.gitee.dbswitch.admin.entity.SystemUserEntity;
import com.gitee.dbswitch.admin.model.response.SystemUserDetailResponse;
import java.util.Objects;

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.controller.converter;
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
import com.gitee.dbswitch.common.converter.AbstractConverter;
import com.gitee.dbswitch.admin.entity.AssignmentJobEntity;
import com.gitee.dbswitch.admin.model.response.TaskJobDetailResponse;
import com.gitee.dbswitch.admin.type.JobStatusEnum;

View File

@@ -20,7 +20,7 @@ import com.gitee.dbswitch.admin.model.request.DbConnectionUpdateRequest;
import com.gitee.dbswitch.admin.model.response.DbConnectionDetailResponse;
import com.gitee.dbswitch.admin.model.response.DbConnectionNameResponse;
import com.gitee.dbswitch.admin.service.ConnectionService;
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
import com.gitee.dbswitch.common.type.ProductTypeEnum;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import javax.annotation.Resource;
@@ -52,7 +52,7 @@ public class ConnectionController {
@TokenCheck
@ApiOperation(value = "数据库类型")
@GetMapping(value = "/{type}/drivers", produces = MediaType.APPLICATION_JSON_VALUE)
public Result getDrivers(@PathVariable("type") SupportDbTypeEnum type) {
public Result getDrivers(@PathVariable("type") ProductTypeEnum type) {
return Result.success(connectionService.getDrivers(type));
}

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.entity;
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
import com.gitee.dbswitch.common.type.ProductTypeEnum;
import java.sql.Timestamp;
import javax.persistence.Column;
import javax.persistence.Entity;
@@ -36,7 +36,7 @@ public class DatabaseConnectionEntity {
private String name;
@Column(name = "type")
private SupportDbTypeEnum type;
private ProductTypeEnum type;
@Column(name = "version")
private String version;

View File

@@ -10,7 +10,7 @@
package com.gitee.dbswitch.admin.model.request;
import com.gitee.dbswitch.admin.entity.DatabaseConnectionEntity;
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
import com.gitee.dbswitch.common.type.ProductTypeEnum;
import lombok.Data;
import lombok.NoArgsConstructor;
@@ -19,7 +19,7 @@ import lombok.NoArgsConstructor;
public class DbConnectionCreateRequest {
private String name;
private SupportDbTypeEnum type;
private ProductTypeEnum type;
private String version;
private String driver;
private String url;
@@ -29,7 +29,7 @@ public class DbConnectionCreateRequest {
public DatabaseConnectionEntity toDatabaseConnection() {
DatabaseConnectionEntity databaseConnectionEntity = new DatabaseConnectionEntity();
databaseConnectionEntity.setId(null);
databaseConnectionEntity.setName(name.trim());
databaseConnectionEntity.setName(name);
databaseConnectionEntity.setType(type);
databaseConnectionEntity.setVersion(version.trim());
databaseConnectionEntity.setDriver(driver.trim());

View File

@@ -10,7 +10,7 @@
package com.gitee.dbswitch.admin.model.request;
import com.gitee.dbswitch.admin.entity.DatabaseConnectionEntity;
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
import com.gitee.dbswitch.common.type.ProductTypeEnum;
import lombok.Data;
import lombok.NoArgsConstructor;
@@ -20,7 +20,7 @@ public class DbConnectionUpdateRequest {
private Long id;
private String name;
private SupportDbTypeEnum type;
private ProductTypeEnum type;
private String version;
private String driver;
private String url;
@@ -30,7 +30,7 @@ public class DbConnectionUpdateRequest {
public DatabaseConnectionEntity toDatabaseConnection() {
DatabaseConnectionEntity databaseConnectionEntity = new DatabaseConnectionEntity();
databaseConnectionEntity.setId(id);
databaseConnectionEntity.setName(name.trim());
databaseConnectionEntity.setName(name);
databaseConnectionEntity.setType(type);
databaseConnectionEntity.setVersion(version.trim());
databaseConnectionEntity.setDriver(driver.trim());

View File

@@ -10,7 +10,7 @@
package com.gitee.dbswitch.admin.model.response;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
import com.gitee.dbswitch.common.type.ProductTypeEnum;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.sql.Timestamp;
@@ -29,7 +29,7 @@ public class DbConnectionDetailResponse {
private String name;
@ApiModelProperty("数据库类型")
private SupportDbTypeEnum type;
private ProductTypeEnum type;
@ApiModelProperty("驱动版本")
private String version;

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.service;
import com.gitee.dbswitch.admin.common.converter.ConverterFactory;
import com.gitee.dbswitch.common.converter.ConverterFactory;
import com.gitee.dbswitch.admin.common.exception.DbswitchException;
import com.gitee.dbswitch.admin.common.response.PageResult;
import com.gitee.dbswitch.admin.common.response.Result;
@@ -27,8 +27,8 @@ import com.gitee.dbswitch.admin.model.request.AssigmentUpdateRequest;
import com.gitee.dbswitch.admin.model.response.AssignmentDetailResponse;
import com.gitee.dbswitch.admin.model.response.AssignmentInfoResponse;
import com.gitee.dbswitch.admin.type.ScheduleModeEnum;
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
import com.gitee.dbswitch.admin.util.PageUtils;
import com.gitee.dbswitch.common.type.ProductTypeEnum;
import com.gitee.dbswitch.data.config.DbswichProperties;
import com.gitee.dbswitch.data.entity.SourceDataSourceProperties;
import com.gitee.dbswitch.data.entity.TargetDataSourceProperties;
@@ -73,11 +73,11 @@ public class AssignmentService {
Long targetConnectionId = assignmentConfigEntity.getTargetConnectionId();
DatabaseConnectionEntity entity = databaseConnectionDAO.getById(targetConnectionId);
if (SupportDbTypeEnum.HIVE == entity.getType()) {
if (ProductTypeEnum.HIVE == entity.getType()) {
throw new DbswitchException(ResultCode.ERROR_INVALID_ASSIGNMENT_CONFIG, "不支持目的端数据源为Hive");
}
if (SupportDbTypeEnum.SQLITE3 == entity.getType()) {
if (SupportDbTypeEnum.isUnsupportedTargetSqlite(entity.getUrl())) {
if (ProductTypeEnum.SQLITE3 == entity.getType()) {
if (ProductTypeEnum.isUnsupportedTargetSqlite(entity.getUrl())) {
throw new DbswitchException(ResultCode.ERROR_INVALID_ASSIGNMENT_CONFIG,
"不支持目的端数据源为远程服务器上的SQLite或内存方式下的SQLite");
}
@@ -111,11 +111,11 @@ public class AssignmentService {
Long targetConnectionId = assignmentConfigEntity.getTargetConnectionId();
DatabaseConnectionEntity entity = databaseConnectionDAO.getById(targetConnectionId);
if (SupportDbTypeEnum.HIVE == entity.getType()) {
if (ProductTypeEnum.HIVE == entity.getType()) {
throw new DbswitchException(ResultCode.ERROR_INVALID_ASSIGNMENT_CONFIG, "不支持目的端数据源为Hive");
}
if (SupportDbTypeEnum.SQLITE3 == entity.getType()) {
if (SupportDbTypeEnum.isUnsupportedTargetSqlite(entity.getUrl())) {
if (ProductTypeEnum.SQLITE3 == entity.getType()) {
if (ProductTypeEnum.isUnsupportedTargetSqlite(entity.getUrl())) {
throw new DbswitchException(ResultCode.ERROR_INVALID_ASSIGNMENT_CONFIG,
"不支持目的端数据源为远程服务器上的SQLite或内存方式下的SQLite");
}

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.service;
import com.gitee.dbswitch.admin.common.converter.ConverterFactory;
import com.gitee.dbswitch.common.converter.ConverterFactory;
import com.gitee.dbswitch.admin.common.exception.DbswitchException;
import com.gitee.dbswitch.admin.common.response.PageResult;
import com.gitee.dbswitch.admin.common.response.Result;
@@ -24,13 +24,13 @@ import com.gitee.dbswitch.admin.model.response.DatabaseTypeDetailResponse;
import com.gitee.dbswitch.admin.model.response.DatabaseTypeDriverResponse;
import com.gitee.dbswitch.admin.model.response.DbConnectionDetailResponse;
import com.gitee.dbswitch.admin.model.response.DbConnectionNameResponse;
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
import com.gitee.dbswitch.admin.util.PageUtils;
import com.gitee.dbswitch.common.entity.CloseableDataSource;
import com.gitee.dbswitch.common.type.ProductTypeEnum;
import com.gitee.dbswitch.common.util.JdbcUrlUtils;
import com.gitee.dbswitch.service.MetadataService;
import com.gitee.dbswitch.service.DefaultMetadataService;
import com.gitee.dbswitch.data.util.DataSourceUtils;
import com.gitee.dbswitch.service.DefaultMetadataService;
import com.gitee.dbswitch.service.MetadataService;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
@@ -53,7 +53,7 @@ public class ConnectionService {
public MetadataService getMetaDataCoreService(DatabaseConnectionEntity dbConn) {
String typeName = dbConn.getType().getName().toUpperCase();
SupportDbTypeEnum supportDbType = SupportDbTypeEnum.valueOf(typeName);
ProductTypeEnum supportDbType = ProductTypeEnum.valueOf(typeName);
if (supportDbType.hasAddress()) {
for (String pattern : supportDbType.getUrl()) {
final Matcher matcher = JdbcUrlUtils.getPattern(pattern).matcher(dbConn.getUrl());
@@ -86,7 +86,7 @@ public class ConnectionService {
public List<DatabaseTypeDetailResponse> getTypes() {
List<DatabaseTypeDetailResponse> lists = new ArrayList<>();
for (SupportDbTypeEnum type : SupportDbTypeEnum.values()) {
for (ProductTypeEnum type : ProductTypeEnum.values()) {
DatabaseTypeDetailResponse detail = new DatabaseTypeDetailResponse();
detail.setId(type.getId());
detail.setType(type.getName().toUpperCase());
@@ -99,7 +99,7 @@ public class ConnectionService {
return lists;
}
public List<DatabaseTypeDriverResponse> getDrivers(SupportDbTypeEnum dbTypeEnum) {
public List<DatabaseTypeDriverResponse> getDrivers(ProductTypeEnum dbTypeEnum) {
List<DatabaseTypeDriverResponse> lists = new ArrayList<>();
driverLoadService.getDriverVersionWithPath(dbTypeEnum)
.forEach(
@@ -256,7 +256,7 @@ public class ConnectionService {
private void validJdbcUrlFormat(DatabaseConnectionEntity conn) {
String typeName = conn.getType().getName().toUpperCase();
SupportDbTypeEnum supportDbType = SupportDbTypeEnum.valueOf(typeName);
ProductTypeEnum supportDbType = ProductTypeEnum.valueOf(typeName);
if (!conn.getUrl().startsWith(supportDbType.getUrlPrefix())) {
throw new DbswitchException(ResultCode.ERROR_INVALID_JDBC_URL, conn.getUrl());
}

View File

@@ -1,7 +1,7 @@
package com.gitee.dbswitch.admin.service;
import com.gitee.dbswitch.admin.config.DbswitchConfig;
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
import com.gitee.dbswitch.common.type.ProductTypeEnum;
import java.io.File;
import java.util.EnumMap;
import java.util.HashMap;
@@ -20,7 +20,7 @@ import org.springframework.stereotype.Service;
@Service
public class DriverLoadService {
private Map<SupportDbTypeEnum, Map<String, File>> drivers = new EnumMap<>(SupportDbTypeEnum.class);
private Map<ProductTypeEnum, Map<String, File>> drivers = new EnumMap<>(ProductTypeEnum.class);
@Resource
private DbswitchConfig dbswitchConfig;
@@ -45,7 +45,7 @@ public class DriverLoadService {
"No drivers type found from path:" + driversBasePath);
}
for (File type : types) {
if (!SupportDbTypeEnum.exists(type.getName())) {
if (!ProductTypeEnum.exists(type.getName())) {
continue;
}
File[] driverVersions = type.listFiles();
@@ -58,7 +58,7 @@ public class DriverLoadService {
throw new IllegalArgumentException(
"No driver version jar file found from path:" + driverVersion.getAbsolutePath());
}
SupportDbTypeEnum typeEnum = SupportDbTypeEnum.of(type.getName());
ProductTypeEnum typeEnum = ProductTypeEnum.of(type.getName());
Map<String, File> versionMap = drivers.computeIfAbsent(typeEnum, k -> new HashMap<>());
versionMap.put(driverVersion.getName(), driverVersion);
log.info("Load driver for {} ,version:{},path:{}",
@@ -67,16 +67,16 @@ public class DriverLoadService {
}
}
public List<String> getDriverVersion(SupportDbTypeEnum dbTypeEnum) {
public List<String> getDriverVersion(ProductTypeEnum dbTypeEnum) {
return Optional.ofNullable(drivers.get(dbTypeEnum)).orElseGet(HashMap::new)
.keySet().stream().collect(Collectors.toList());
}
public Map<String, File> getDriverVersionWithPath(SupportDbTypeEnum dbTypeEnum) {
public Map<String, File> getDriverVersionWithPath(ProductTypeEnum dbTypeEnum) {
return Optional.ofNullable(drivers.get(dbTypeEnum)).orElse(new HashMap<>());
}
public File getVersionDriverFile(SupportDbTypeEnum dbTypeEnum, String driverVersion) {
public File getVersionDriverFile(ProductTypeEnum dbTypeEnum, String driverVersion) {
return drivers.get(dbTypeEnum).get(driverVersion);
}

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.service;
import com.gitee.dbswitch.admin.common.converter.ConverterFactory;
import com.gitee.dbswitch.common.converter.ConverterFactory;
import com.gitee.dbswitch.admin.common.response.PageResult;
import com.gitee.dbswitch.admin.common.response.Result;
import com.gitee.dbswitch.admin.common.response.ResultCode;

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.service;
import com.gitee.dbswitch.admin.common.converter.ConverterFactory;
import com.gitee.dbswitch.common.converter.ConverterFactory;
import com.gitee.dbswitch.admin.common.response.PageResult;
import com.gitee.dbswitch.admin.common.response.Result;
import com.gitee.dbswitch.admin.common.response.ResultCode;

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.service;
import com.gitee.dbswitch.admin.common.converter.ConverterFactory;
import com.gitee.dbswitch.common.converter.ConverterFactory;
import com.gitee.dbswitch.admin.common.response.Result;
import com.gitee.dbswitch.admin.common.response.ResultCode;
import com.gitee.dbswitch.admin.controller.converter.SystemUserDetailConverter;

View File

@@ -1,119 +0,0 @@
// Copyright tang. All rights reserved.
// https://gitee.com/inrgihc/dbswitch
//
// Use of this source code is governed by a BSD-style license
//
// Author: tang (inrgihc@126.com)
// Date : 2020/1/2
// Location: beijing , china
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.type;
import java.util.Arrays;
import lombok.AllArgsConstructor;
import lombok.Getter;
import org.springframework.util.StringUtils;
@Getter
@AllArgsConstructor
public enum SupportDbTypeEnum {
MYSQL(1, "mysql", "com.mysql.jdbc.Driver", 3306,
"/* ping */ SELECT 1",
"jdbc:mysql://",
new String[]{"jdbc:mysql://{host}[:{port}]/[{database}][\\?{params}]"}),
MARIADB(2, "mariadb", "org.mariadb.jdbc.Driver", 3306,
"SELECT 1",
"jdbc:mariadb://",
new String[]{"jdbc:mariadb://{host}[:{port}]/[{database}][\\?{params}]"}),
ORACLE(3, "oracle", "oracle.jdbc.driver.OracleDriver", 1521,
"SELECT 'Hello' from DUAL",
"jdbc:oracle:thin:@",
new String[]{"jdbc:oracle:thin:@{host}:{port}:{database}",
"jdbc:oracle:thin:@//{host}[:{port}]/{database}"}),
SQLSERVER(4, "sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", 1433,
"SELECT 1+2 as a",
"jdbc:sqlserver://",
new String[]{"jdbc:sqlserver://{host}[:{port}][;DatabaseName={database}][;{params}]"}),
POSTGRESQL(5, "postgresql", "org.postgresql.Driver", 5432,
"SELECT 1",
"jdbc:postgresql://",
new String[]{"jdbc:postgresql://{host}[:{port}]/[{database}][\\?{params}]"}),
DB2(6, "db2", "com.ibm.db2.jcc.DB2Driver", 50000,
"SELECT 1 FROM SYSIBM.SYSDUMMY1",
"jdbc:db2://",
new String[]{"jdbc:db2://{host}:{port}/{database}[:{params}]"}),
DM(7, "dm", "dm.jdbc.driver.DmDriver", 5236,
"SELECT 'Hello' from DUAL",
"jdbc:dm://",
new String[]{"jdbc:dm://{host}:{port}[/{database}][\\?{params}]"}),
KINGBASE(8, "kingbase", "com.kingbase8.Driver", 54321,
"SELECT 1",
"jdbc:kingbase8://",
new String[]{"jdbc:kingbase8://{host}[:{port}]/[{database}][\\?{params}]"}),
OSCAR(9, "oscar", "com.oscar.Driver", 2003,
"SELECT 1",
"jdbc:oscar://",
new String[]{"jdbc:oscar://{host}[:{port}]/[{database}][\\?{params}]"}),
GBASE8A(10, "gbase8a", "com.gbase.jdbc.Driver", 5258,
"/* ping */ SELECT 1",
"jdbc:gbase://",
new String[]{"jdbc:gbase://{host}[:{port}]/[{database}][\\?{params}]"}),
SYBASE(11, "sybase", "com.sybase.jdbc4.jdbc.SybDriver", 5000,
"SELECT 1+2 as a",
"jdbc:sybase:Tds:",
new String[]{"jdbc:sybase:Tds:{host}[:{port}][/{database}][\\?{params}]"}),
HIVE(12, "hive", "org.apache.hive.jdbc.HiveDriver", 10000,
"SELECT 1",
"jdbc:hive2://",
new String[]{"jdbc:hive2://{host}[:{port}]/[{database}][\\?{params}]"}),
// 参考文章https://blog.csdn.net/wank1259162/article/details/104946744
SQLITE3(13, "sqlite3", "org.sqlite.JDBC", 0,
"SELECT 1",
"jdbc:sqlite:",
new String[]{"jdbc:sqlite:{file}", "jdbc:sqlite::resource:{file}"}),
;
private int id;
private String name;
private String driver;
private int port;
private String sql;
private String urlPrefix;
private String[] url;
public boolean hasDatabaseName() {
return !Arrays.asList(DM, SQLITE3).contains(this);
}
public boolean hasFilePath() {
return this == SQLITE3;
}
public boolean hasAddress() {
return this != SQLITE3;
}
public static boolean exists(String name) {
return Arrays.stream(values()).anyMatch(item -> item.name().equalsIgnoreCase(name));
}
public static SupportDbTypeEnum of(String name) {
if (!StringUtils.isEmpty(name)) {
for (SupportDbTypeEnum type : SupportDbTypeEnum.values()) {
if (type.getName().equalsIgnoreCase(name)) {
return type;
}
}
}
throw new IllegalArgumentException("cannot find enum name: " + name);
}
public static boolean isUnsupportedTargetSqlite(String url) {
String prefix1 = "jdbc:sqlite::resource:";
//String prefix2 = "jdbc:sqlite::memory:";
return url.startsWith(prefix1);
}
}

View File

@@ -7,7 +7,7 @@
// Date : 2020/1/2
// Location: beijing , china
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.common.converter;
package com.gitee.dbswitch.common.converter;
import java.util.List;
import java.util.Set;

View File

@@ -7,7 +7,7 @@
// Date : 2020/1/2
// Location: beijing , china
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.common.converter;
package com.gitee.dbswitch.common.converter;
public interface Converter<U, V> {

View File

@@ -7,7 +7,7 @@
// Date : 2020/1/2
// Location: beijing , china
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.admin.common.converter;
package com.gitee.dbswitch.common.converter;
import java.util.Map;
import java.util.Objects;

View File

@@ -10,104 +10,145 @@
package com.gitee.dbswitch.common.type;
import java.util.Arrays;
import lombok.AllArgsConstructor;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
/**
* 数据库产品类型的枚举定义
*
* @author Tang
*/
@Getter
@AllArgsConstructor
public enum ProductTypeEnum {
/**
* 未知数据库类型
*/
UNKNOWN(0, "\""),
/**
* MySQL数据库类型
*/
MYSQL(1, "`"),
/**
* Oracle数据库类型
*/
ORACLE(2, "\""),
/**
* SQLServer 2000数据库类型
*/
SQLSERVER2000(3, "\""),
/**
* SQLServer数据库类型
*/
SQLSERVER(4, "\""),
/**
* PostgreSQL数据库类型
*/
POSTGRESQL(5, "\""),
/**
* Greenplum数据库类型
*/
GREENPLUM(6, "\""),
MYSQL(1, "`", "mysql", "com.mysql.jdbc.Driver", 3306,
"/* ping */ SELECT 1",
"jdbc:mysql://",
new String[]{"jdbc:mysql://{host}[:{port}]/[{database}][\\?{params}]"}),
/**
* MariaDB数据库类型
*/
MARIADB(7, "`"),
MARIADB(2, "`", "mariadb", "org.mariadb.jdbc.Driver", 3306,
"SELECT 1",
"jdbc:mariadb://",
new String[]{"jdbc:mariadb://{host}[:{port}]/[{database}][\\?{params}]"}),
/**
* Oracle数据库类型
*/
ORACLE(3, "\"", "oracle", "oracle.jdbc.driver.OracleDriver", 1521,
"SELECT 'Hello' from DUAL",
"jdbc:oracle:thin:@",
new String[]{"jdbc:oracle:thin:@{host}:{port}:{database}",
"jdbc:oracle:thin:@//{host}[:{port}]/{database}"}),
/**
* Microsoft SQL Server数据库类型(>=2005)
*/
SQLSERVER(4, "\"", "sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", 1433,
"SELECT 1+2 as a",
"jdbc:sqlserver://",
new String[]{"jdbc:sqlserver://{host}[:{port}][;DatabaseName={database}][;{params}]"}),
/**
* PostgreSQL数据库类型
*/
POSTGRESQL(5, "\"", "postgresql", "org.postgresql.Driver", 5432,
"SELECT 1",
"jdbc:postgresql://",
new String[]{"jdbc:postgresql://{host}[:{port}]/[{database}][\\?{params}]"}),
/**
* DB2数据库类型
*/
DB2(8, "\""),
DB2(6, "\"", "db2", "com.ibm.db2.jcc.DB2Driver", 50000,
"SELECT 1 FROM SYSIBM.SYSDUMMY1",
"jdbc:db2://",
new String[]{"jdbc:db2://{host}:{port}/{database}[:{params}]"}),
/**
* [国产]达梦数据库类型
* [国产] 达梦(DM)数据库类型
*/
DM(9, "\""),
DM(7, "\"", "dm", "dm.jdbc.driver.DmDriver", 5236,
"SELECT 'Hello' from DUAL",
"jdbc:dm://",
new String[]{"jdbc:dm://{host}:{port}[/{database}][\\?{params}]"}),
/**
* [国产]人大金仓数据库类型
* [国产] 金仓(Kingbase)数据库类型
*/
KINGBASE(10, "\""),
KINGBASE(8, "\"", "kingbase", "com.kingbase8.Driver", 54321,
"SELECT 1",
"jdbc:kingbase8://",
new String[]{"jdbc:kingbase8://{host}[:{port}]/[{database}][\\?{params}]"}),
/**
* [国产]神通数据库
* [国产] 神通(Oscar)数据库类型
*/
OSCAR(11, "\""),
OSCAR(9, "\"", "oscar", "com.oscar.Driver", 2003,
"SELECT 1",
"jdbc:oscar://",
new String[]{"jdbc:oscar://{host}[:{port}]/[{database}][\\?{params}]"}),
/**
* [国产]南大通用GBase8a数据库
* [国产] 南大通用(GBase8A)数据库类型
*/
GBASE8A(12, "`"),
GBASE8A(10, "`", "gbase8a", "com.gbase.jdbc.Driver", 5258,
"/* ping */ SELECT 1",
"jdbc:gbase://",
new String[]{"jdbc:gbase://{host}[:{port}]/[{database}][\\?{params}]"}),
/**
* HIVE数据库
* Sybase 数据库类型
*/
HIVE(13, "`"),
SYBASE(11, "\"", "sybase", "com.sybase.jdbc4.jdbc.SybDriver", 5000,
"SELECT 1+2 as a",
"jdbc:sybase:Tds:",
new String[]{"jdbc:sybase:Tds:{host}[:{port}][/{database}][\\?{params}]"}),
/**
* SQLite数据库
* Hive 数据库类型
*/
SQLITE3(14, "\""),
HIVE(12, "`", "hive", "org.apache.hive.jdbc.HiveDriver", 10000,
"SELECT 1",
"jdbc:hive2://",
new String[]{"jdbc:hive2://{host}[:{port}]/[{database}][\\?{params}]"}),
/**
* Sybase数据库类型
* Sqlite v3数据库类型
*/
SYBASE(15, "\""),
// 参考文章https://blog.csdn.net/wank1259162/article/details/104946744
SQLITE3(13, "\"", "sqlite3", "org.sqlite.JDBC", 0,
"SELECT 1",
"jdbc:sqlite:",
new String[]{"jdbc:sqlite:{file}", "jdbc:sqlite::resource:{file}"}),
;
private int index;
private int id;
private String quote;
private String name;
private String driver;
private int port;
private String sql;
private String urlPrefix;
private String[] url;
ProductTypeEnum(int idx, String quote) {
this.index = idx;
this.quote = quote;
public boolean hasDatabaseName() {
return !Arrays.asList(DM, SQLITE3).contains(this);
}
public int getIndex() {
return index;
public boolean hasFilePath() {
return this == SQLITE3;
}
public boolean hasAddress() {
return this != SQLITE3;
}
public boolean noCommentStatement() {
@@ -129,4 +170,26 @@ public enum ProductTypeEnum {
return String.format("%s%s%s.%s%s%s", quote, schema, quote, quote, table, quote);
}
public static boolean exists(String name) {
return Arrays.stream(values()).anyMatch(item -> item.name().equalsIgnoreCase(name));
}
public static ProductTypeEnum of(String name) {
if (!StringUtils.isEmpty(name)) {
for (ProductTypeEnum type : ProductTypeEnum.values()) {
if (type.getName().equalsIgnoreCase(name)) {
return type;
}
}
}
throw new IllegalArgumentException("cannot find enum name: " + name);
}
public static boolean isUnsupportedTargetSqlite(String url) {
String prefix1 = "jdbc:sqlite::resource:";
//String prefix2 = "jdbc:sqlite::memory:";
return url.startsWith(prefix1);
}
}

View File

@@ -35,7 +35,6 @@ public final class DatabaseAwareUtils {
productNameMap = new HashMap<>();
driverNameMap = new HashMap<>();
productNameMap.put("Greenplum", ProductTypeEnum.GREENPLUM);
productNameMap.put("Microsoft SQL Server", ProductTypeEnum.SQLSERVER);
productNameMap.put("DM DBMS", ProductTypeEnum.DM);
productNameMap.put("KingbaseES", ProductTypeEnum.KINGBASE);
@@ -116,5 +115,5 @@ public final class DatabaseAwareUtils {
throw new RuntimeException(se);
}
}
}

View File

@@ -9,6 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.common.util;
import cn.hutool.core.convert.Convert;
import java.lang.reflect.Field;
import java.sql.Types;
import java.util.HashMap;
@@ -134,5 +135,20 @@ public final class JdbcTypesUtils {
// REF
// DATALINK
// REF_CURSOR
public static long getObjectSize(int jdbcType, Object value) {
if (null == value) {
return 0;
}
if (isBinary(jdbcType)) {
byte[] bytes = Convert.toPrimitiveByteArray(value);
return null == bytes ? 0 : bytes.length;
} else if (isBoolean(jdbcType)) {
return 1;
} else {
String strValue = Convert.toStr(value);
return null == strValue ? 0 : strValue.length();
}
}
}

View File

@@ -15,8 +15,8 @@ import com.gitee.dbswitch.common.util.JdbcTypesUtils;
import com.gitee.dbswitch.common.util.TypeConvertUtils;
import com.gitee.dbswitch.provider.ProductProviderFactory;
import com.gitee.dbswitch.provider.query.TableDataQueryProvider;
import com.gitee.dbswitch.service.MetadataService;
import com.gitee.dbswitch.service.DefaultMetadataService;
import com.gitee.dbswitch.service.MetadataService;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
@@ -268,6 +268,7 @@ public final class ChangeCalculatorService implements IDatabaseChangeCalculator
}
// 初始化计算结果数据字段列信息
int[] jdbcTypes = new int[metaData.getColumnCount()];
List<String> targetColumns = new ArrayList<>();
for (int k = 1; k <= metaData.getColumnCount(); ++k) {
String key = metaData.getColumnLabel(k);
@@ -275,6 +276,7 @@ public final class ChangeCalculatorService implements IDatabaseChangeCalculator
key = metaData.getColumnName(k);
}
targetColumns.add(columnsMap.getOrDefault(key, key));
jdbcTypes[k - 1] = metaData.getColumnType(k);
}
if (log.isDebugEnabled()) {
@@ -329,7 +331,7 @@ public final class ChangeCalculatorService implements IDatabaseChangeCalculator
}
// 这里对计算的单条记录结果进行处理
handler.handle(Collections.unmodifiableList(targetColumns), outputRow, flagField);
handler.handle(Collections.unmodifiableList(targetColumns), outputRow, jdbcTypes, flagField);
}
if (log.isDebugEnabled()) {

View File

@@ -21,11 +21,12 @@ public interface IDatabaseRowHandler {
/**
* 行数据处理
*
* @param fields 字段名称列表,该列表只读
* @param record 一条数据记实录
* @param flag 数据变化状态
* @param fields 字段名称列表,该列表只读
* @param record 一条数据记实录
* @param jdbcTypes jdbc类型
* @param flag 数据变化状态
*/
void handle(List<String> fields, Object[] record, RecordChangeTypeEnum flag);
void handle(List<String> fields, Object[] record, int[] jdbcTypes, RecordChangeTypeEnum flag);
/**
* 计算结束通知

View File

@@ -350,7 +350,6 @@ public class ColumnMetaData {
// If we're dealing with PostgreSQL and double precision types
if ((desc.getProductType() == ProductTypeEnum.POSTGRESQL
|| desc.getProductType() == ProductTypeEnum.GREENPLUM
|| desc.getProductType() == ProductTypeEnum.KINGBASE)
&& type == java.sql.Types.DOUBLE
&& precision >= 16
@@ -402,8 +401,7 @@ public class ColumnMetaData {
}
if (desc.getProductType() == ProductTypeEnum.POSTGRESQL
|| desc.getProductType() == ProductTypeEnum.KINGBASE
|| desc.getProductType() == ProductTypeEnum.GREENPLUM) {
|| desc.getProductType() == ProductTypeEnum.KINGBASE) {
// undefined size => arbitrary precision
if (type == java.sql.Types.NUMERIC && length == 0 && precision == 0) {
valtype = ColumnMetaData.TYPE_BIGNUMBER;

View File

@@ -35,11 +35,6 @@
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>org.ehcache</groupId>
<artifactId>sizeof</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>

View File

@@ -9,7 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.data.domain;
import com.gitee.dbswitch.data.util.BytesUnitUtils;
import cn.hutool.core.io.unit.DataSizeUtil;
import lombok.AllArgsConstructor;
import lombok.Data;
@@ -32,6 +32,6 @@ public class PerfStat {
return "Data Source Index: \t" + index + "\n" +
"Total Tables Count: \t" + total + "\n" +
"Failure Tables count: \t" + failure + "\n" +
"Total Transfer Size: \t" + BytesUnitUtils.bytesSizeToHuman(bytes) + "\n";
"Total Transfer Size: \t" + DataSizeUtil.format(bytes) + "\n";
}
}

View File

@@ -9,6 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.data.handler;
import cn.hutool.core.io.unit.DataSizeUtil;
import com.gitee.dbswitch.calculate.ChangeCalculatorService;
import com.gitee.dbswitch.calculate.IDatabaseChangeCalculator;
import com.gitee.dbswitch.calculate.IDatabaseRowHandler;
@@ -18,10 +19,10 @@ import com.gitee.dbswitch.common.entity.CloseableDataSource;
import com.gitee.dbswitch.common.entity.ResultSetWrapper;
import com.gitee.dbswitch.common.type.ProductTypeEnum;
import com.gitee.dbswitch.common.util.DatabaseAwareUtils;
import com.gitee.dbswitch.common.util.JdbcTypesUtils;
import com.gitee.dbswitch.common.util.PatterNameUtils;
import com.gitee.dbswitch.data.config.DbswichProperties;
import com.gitee.dbswitch.data.entity.SourceDataSourceProperties;
import com.gitee.dbswitch.data.util.BytesUnitUtils;
import com.gitee.dbswitch.provider.ProductFactoryProvider;
import com.gitee.dbswitch.provider.ProductProviderFactory;
import com.gitee.dbswitch.provider.meta.MetadataProvider;
@@ -31,9 +32,10 @@ import com.gitee.dbswitch.provider.sync.TableDataSynchronizer;
import com.gitee.dbswitch.provider.write.TableDataWriteProvider;
import com.gitee.dbswitch.schema.ColumnDescription;
import com.gitee.dbswitch.schema.TableDescription;
import com.gitee.dbswitch.service.MetadataService;
import com.gitee.dbswitch.service.DefaultMetadataService;
import com.gitee.dbswitch.service.MetadataService;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -45,7 +47,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.ehcache.sizeof.SizeOf;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.util.StringUtils;
@@ -57,7 +58,7 @@ import org.springframework.util.StringUtils;
@Slf4j
public class MigrationHandler implements Supplier<Long> {
private final long MAX_CACHE_BYTES_SIZE = 512 * 1024 * 1024;
private final long MAX_CACHE_BYTES_SIZE = 128 * 1024 * 1024;
private int fetchSize = 100;
private final DbswichProperties properties;
@@ -360,15 +361,19 @@ public class MigrationHandler implements Supplier<Long> {
long totalCount = 0;
long totalBytes = 0;
try (ResultSet rs = srs.getResultSet()) {
ResultSetMetaData metaData = rs.getMetaData();
while (rs.next()) {
if (interrupted) {
log.info("task job is interrupted!");
throw new RuntimeException("task is interrupted");
}
Object[] record = new Object[sourceFields.size()];
long bytes = 0;
for (int i = 1; i <= sourceFields.size(); ++i) {
try {
record[i - 1] = rs.getObject(i);
Object value = rs.getObject(i);
bytes += JdbcTypesUtils.getObjectSize(metaData.getColumnType(i), value);
record[i - 1] = value;
} catch (Exception e) {
log.warn("!!! Read data from table [ {} ] use function ResultSet.getObject() error",
tableNameMapString, e);
@@ -377,14 +382,13 @@ public class MigrationHandler implements Supplier<Long> {
}
cache.add(record);
long bytes = SizeOf.newInstance().deepSizeOf(record);
cacheBytes += bytes;
++totalCount;
if (cache.size() >= BATCH_SIZE || cacheBytes >= MAX_CACHE_BYTES_SIZE) {
long ret = writer.write(targetFields, cache);
log.info("[FullCoverSync] handle table [{}] data count: {}, the batch bytes sie: {}",
tableNameMapString, ret, BytesUnitUtils.bytesSizeToHuman(cacheBytes));
tableNameMapString, ret, DataSizeUtil.format(cacheBytes));
cache.clear();
totalBytes += cacheBytes;
cacheBytes = 0;
@@ -394,13 +398,13 @@ public class MigrationHandler implements Supplier<Long> {
if (cache.size() > 0) {
long ret = writer.write(targetFields, cache);
log.info("[FullCoverSync] handle table [{}] data count: {}, last batch bytes sie: {}",
tableNameMapString, ret, BytesUnitUtils.bytesSizeToHuman(cacheBytes));
tableNameMapString, ret, DataSizeUtil.format(cacheBytes));
cache.clear();
totalBytes += cacheBytes;
}
log.info("[FullCoverSync] handle table [{}] total data count:{}, total bytes={}",
tableNameMapString, totalCount, BytesUnitUtils.bytesSizeToHuman(totalBytes));
tableNameMapString, totalCount, DataSizeUtil.format(totalBytes));
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
@@ -465,7 +469,7 @@ public class MigrationHandler implements Supplier<Long> {
private final List<Object[]> cacheDelete = new LinkedList<>();
@Override
public void handle(List<String> fields, Object[] record, RecordChangeTypeEnum flag) {
public void handle(List<String> fields, Object[] record, int[] jdbcTypes, RecordChangeTypeEnum flag) {
if (flag == RecordChangeTypeEnum.VALUE_INSERT) {
cacheInsert.add(record);
countInsert++;
@@ -477,7 +481,11 @@ public class MigrationHandler implements Supplier<Long> {
countDelete++;
}
long bytes = SizeOf.newInstance().deepSizeOf(record);
long bytes = 0;
for (int i = 0; i < record.length; ++i) {
Object value = record[i];
bytes += JdbcTypesUtils.getObjectSize(jdbcTypes[i], value);
}
cacheBytes += bytes;
totalBytes.addAndGet(bytes);
countTotal++;
@@ -509,7 +517,7 @@ public class MigrationHandler implements Supplier<Long> {
}
log.info("[IncreaseSync] Handle table [{}] data one batch size: {}",
tableNameMapString, BytesUnitUtils.bytesSizeToHuman(cacheBytes));
tableNameMapString, DataSizeUtil.format(cacheBytes));
cacheBytes = 0;
}
}

View File

@@ -9,6 +9,7 @@
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.data.service;
import cn.hutool.core.io.unit.DataSizeUtil;
import cn.hutool.core.stream.StreamUtil;
import cn.hutool.core.text.StrPool;
import cn.hutool.core.util.StrUtil;
@@ -21,12 +22,11 @@ import com.gitee.dbswitch.data.config.DbswichProperties;
import com.gitee.dbswitch.data.domain.PerfStat;
import com.gitee.dbswitch.data.entity.SourceDataSourceProperties;
import com.gitee.dbswitch.data.handler.MigrationHandler;
import com.gitee.dbswitch.data.util.BytesUnitUtils;
import com.gitee.dbswitch.data.util.DataSourceUtils;
import com.gitee.dbswitch.data.util.JsonUtils;
import com.gitee.dbswitch.schema.TableDescription;
import com.gitee.dbswitch.service.MetadataService;
import com.gitee.dbswitch.service.DefaultMetadataService;
import com.gitee.dbswitch.service.MetadataService;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -222,7 +222,7 @@ public class MigrationService {
log.info(
"#### Complete data migration for the [ {} ] data source:\ntotal count={}\nfailure count={}\ntotal bytes size={}",
sourcePropertiesIndex, futures.size(), numberOfFailures.get(),
BytesUnitUtils.bytesSizeToHuman(totalBytesSize.get()));
DataSizeUtil.format(totalBytesSize.get()));
perfStats.add(new PerfStat(sourcePropertiesIndex, futures.size(),
numberOfFailures.get(), totalBytesSize.get()));
++sourcePropertiesIndex;

View File

@@ -1,51 +0,0 @@
// Copyright tang. All rights reserved.
// https://gitee.com/inrgihc/dbswitch
//
// Use of this source code is governed by a BSD-style license
//
// Author: tang (inrgihc@126.com)
// Date : 2020/1/2
// Location: beijing , china
/////////////////////////////////////////////////////////////
package com.gitee.dbswitch.data.util;
import java.text.DecimalFormat;
import lombok.experimental.UtilityClass;
/**
* 字节单位转换
*
* @author tang
*/
@UtilityClass
public final class BytesUnitUtils {
public static String bytesSizeToHuman(long size) {
/** 定义GB的计算常量 */
long GB = 1024 * 1024 * 1024;
/** 定义MB的计算常量 */
long MB = 1024 * 1024;
/** 定义KB的计算常量 */
long KB = 1024;
/** 格式化小数 */
DecimalFormat df = new DecimalFormat("0.00");
String resultSize = "0.00";
if (size / GB >= 1) {
//如果当前Byte的值大于等于1GB
resultSize = df.format(size / (float) GB) + "GB ";
} else if (size / MB >= 1) {
//如果当前Byte的值大于等于1MB
resultSize = df.format(size / (float) MB) + "MB ";
} else if (size / KB >= 1) {
//如果当前Byte的值大于等于1KB
resultSize = df.format(size / (float) KB) + "KB ";
} else {
resultSize = size + "B ";
}
return resultSize;
}
}

View File

@@ -62,12 +62,6 @@
<version>3.6</version>
</dependency>
<dependency>
<groupId>org.ehcache</groupId>
<artifactId>sizeof</artifactId>
<version>0.4.0</version>
</dependency>
<dependency>
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>