mirror of
https://gitee.com/dromara/dbswitch.git
synced 2025-09-04 11:25:44 +00:00
dbswitch模块代码梳理调整
This commit is contained in:
@@ -10,7 +10,7 @@
|
||||
package com.gitee.dbswitch.admin.controller.converter;
|
||||
|
||||
import cn.hutool.extra.spring.SpringUtil;
|
||||
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.admin.dao.AssignmentConfigDAO;
|
||||
import com.gitee.dbswitch.admin.dao.DatabaseConnectionDAO;
|
||||
import com.gitee.dbswitch.admin.entity.AssignmentConfigEntity;
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.controller.converter;
|
||||
|
||||
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.admin.entity.AssignmentTaskEntity;
|
||||
import com.gitee.dbswitch.admin.model.response.AssignmentInfoResponse;
|
||||
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.controller.converter;
|
||||
|
||||
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.admin.entity.DatabaseConnectionEntity;
|
||||
import com.gitee.dbswitch.admin.model.response.DbConnectionDetailResponse;
|
||||
import java.util.Objects;
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.controller.converter;
|
||||
|
||||
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.admin.entity.SystemLogEntity;
|
||||
import com.gitee.dbswitch.admin.model.response.SystemLogDetailResponse;
|
||||
import java.util.Objects;
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.controller.converter;
|
||||
|
||||
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.admin.entity.SystemUserEntity;
|
||||
import com.gitee.dbswitch.admin.model.response.SystemUserDetailResponse;
|
||||
import java.util.Objects;
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.controller.converter;
|
||||
|
||||
import com.gitee.dbswitch.admin.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.common.converter.AbstractConverter;
|
||||
import com.gitee.dbswitch.admin.entity.AssignmentJobEntity;
|
||||
import com.gitee.dbswitch.admin.model.response.TaskJobDetailResponse;
|
||||
import com.gitee.dbswitch.admin.type.JobStatusEnum;
|
||||
|
@@ -20,7 +20,7 @@ import com.gitee.dbswitch.admin.model.request.DbConnectionUpdateRequest;
|
||||
import com.gitee.dbswitch.admin.model.response.DbConnectionDetailResponse;
|
||||
import com.gitee.dbswitch.admin.model.response.DbConnectionNameResponse;
|
||||
import com.gitee.dbswitch.admin.service.ConnectionService;
|
||||
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
|
||||
import com.gitee.dbswitch.common.type.ProductTypeEnum;
|
||||
import io.swagger.annotations.Api;
|
||||
import io.swagger.annotations.ApiOperation;
|
||||
import javax.annotation.Resource;
|
||||
@@ -52,7 +52,7 @@ public class ConnectionController {
|
||||
@TokenCheck
|
||||
@ApiOperation(value = "数据库类型")
|
||||
@GetMapping(value = "/{type}/drivers", produces = MediaType.APPLICATION_JSON_VALUE)
|
||||
public Result getDrivers(@PathVariable("type") SupportDbTypeEnum type) {
|
||||
public Result getDrivers(@PathVariable("type") ProductTypeEnum type) {
|
||||
return Result.success(connectionService.getDrivers(type));
|
||||
}
|
||||
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.entity;
|
||||
|
||||
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
|
||||
import com.gitee.dbswitch.common.type.ProductTypeEnum;
|
||||
import java.sql.Timestamp;
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
@@ -36,7 +36,7 @@ public class DatabaseConnectionEntity {
|
||||
private String name;
|
||||
|
||||
@Column(name = "type")
|
||||
private SupportDbTypeEnum type;
|
||||
private ProductTypeEnum type;
|
||||
|
||||
@Column(name = "version")
|
||||
private String version;
|
||||
|
@@ -10,7 +10,7 @@
|
||||
package com.gitee.dbswitch.admin.model.request;
|
||||
|
||||
import com.gitee.dbswitch.admin.entity.DatabaseConnectionEntity;
|
||||
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
|
||||
import com.gitee.dbswitch.common.type.ProductTypeEnum;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@@ -19,7 +19,7 @@ import lombok.NoArgsConstructor;
|
||||
public class DbConnectionCreateRequest {
|
||||
|
||||
private String name;
|
||||
private SupportDbTypeEnum type;
|
||||
private ProductTypeEnum type;
|
||||
private String version;
|
||||
private String driver;
|
||||
private String url;
|
||||
@@ -29,7 +29,7 @@ public class DbConnectionCreateRequest {
|
||||
public DatabaseConnectionEntity toDatabaseConnection() {
|
||||
DatabaseConnectionEntity databaseConnectionEntity = new DatabaseConnectionEntity();
|
||||
databaseConnectionEntity.setId(null);
|
||||
databaseConnectionEntity.setName(name.trim());
|
||||
databaseConnectionEntity.setName(name);
|
||||
databaseConnectionEntity.setType(type);
|
||||
databaseConnectionEntity.setVersion(version.trim());
|
||||
databaseConnectionEntity.setDriver(driver.trim());
|
||||
|
@@ -10,7 +10,7 @@
|
||||
package com.gitee.dbswitch.admin.model.request;
|
||||
|
||||
import com.gitee.dbswitch.admin.entity.DatabaseConnectionEntity;
|
||||
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
|
||||
import com.gitee.dbswitch.common.type.ProductTypeEnum;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@@ -20,7 +20,7 @@ public class DbConnectionUpdateRequest {
|
||||
|
||||
private Long id;
|
||||
private String name;
|
||||
private SupportDbTypeEnum type;
|
||||
private ProductTypeEnum type;
|
||||
private String version;
|
||||
private String driver;
|
||||
private String url;
|
||||
@@ -30,7 +30,7 @@ public class DbConnectionUpdateRequest {
|
||||
public DatabaseConnectionEntity toDatabaseConnection() {
|
||||
DatabaseConnectionEntity databaseConnectionEntity = new DatabaseConnectionEntity();
|
||||
databaseConnectionEntity.setId(id);
|
||||
databaseConnectionEntity.setName(name.trim());
|
||||
databaseConnectionEntity.setName(name);
|
||||
databaseConnectionEntity.setType(type);
|
||||
databaseConnectionEntity.setVersion(version.trim());
|
||||
databaseConnectionEntity.setDriver(driver.trim());
|
||||
|
@@ -10,7 +10,7 @@
|
||||
package com.gitee.dbswitch.admin.model.response;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
|
||||
import com.gitee.dbswitch.common.type.ProductTypeEnum;
|
||||
import io.swagger.annotations.ApiModel;
|
||||
import io.swagger.annotations.ApiModelProperty;
|
||||
import java.sql.Timestamp;
|
||||
@@ -29,7 +29,7 @@ public class DbConnectionDetailResponse {
|
||||
private String name;
|
||||
|
||||
@ApiModelProperty("数据库类型")
|
||||
private SupportDbTypeEnum type;
|
||||
private ProductTypeEnum type;
|
||||
|
||||
@ApiModelProperty("驱动版本")
|
||||
private String version;
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.service;
|
||||
|
||||
import com.gitee.dbswitch.admin.common.converter.ConverterFactory;
|
||||
import com.gitee.dbswitch.common.converter.ConverterFactory;
|
||||
import com.gitee.dbswitch.admin.common.exception.DbswitchException;
|
||||
import com.gitee.dbswitch.admin.common.response.PageResult;
|
||||
import com.gitee.dbswitch.admin.common.response.Result;
|
||||
@@ -27,8 +27,8 @@ import com.gitee.dbswitch.admin.model.request.AssigmentUpdateRequest;
|
||||
import com.gitee.dbswitch.admin.model.response.AssignmentDetailResponse;
|
||||
import com.gitee.dbswitch.admin.model.response.AssignmentInfoResponse;
|
||||
import com.gitee.dbswitch.admin.type.ScheduleModeEnum;
|
||||
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
|
||||
import com.gitee.dbswitch.admin.util.PageUtils;
|
||||
import com.gitee.dbswitch.common.type.ProductTypeEnum;
|
||||
import com.gitee.dbswitch.data.config.DbswichProperties;
|
||||
import com.gitee.dbswitch.data.entity.SourceDataSourceProperties;
|
||||
import com.gitee.dbswitch.data.entity.TargetDataSourceProperties;
|
||||
@@ -73,11 +73,11 @@ public class AssignmentService {
|
||||
|
||||
Long targetConnectionId = assignmentConfigEntity.getTargetConnectionId();
|
||||
DatabaseConnectionEntity entity = databaseConnectionDAO.getById(targetConnectionId);
|
||||
if (SupportDbTypeEnum.HIVE == entity.getType()) {
|
||||
if (ProductTypeEnum.HIVE == entity.getType()) {
|
||||
throw new DbswitchException(ResultCode.ERROR_INVALID_ASSIGNMENT_CONFIG, "不支持目的端数据源为Hive");
|
||||
}
|
||||
if (SupportDbTypeEnum.SQLITE3 == entity.getType()) {
|
||||
if (SupportDbTypeEnum.isUnsupportedTargetSqlite(entity.getUrl())) {
|
||||
if (ProductTypeEnum.SQLITE3 == entity.getType()) {
|
||||
if (ProductTypeEnum.isUnsupportedTargetSqlite(entity.getUrl())) {
|
||||
throw new DbswitchException(ResultCode.ERROR_INVALID_ASSIGNMENT_CONFIG,
|
||||
"不支持目的端数据源为远程服务器上的SQLite或内存方式下的SQLite");
|
||||
}
|
||||
@@ -111,11 +111,11 @@ public class AssignmentService {
|
||||
|
||||
Long targetConnectionId = assignmentConfigEntity.getTargetConnectionId();
|
||||
DatabaseConnectionEntity entity = databaseConnectionDAO.getById(targetConnectionId);
|
||||
if (SupportDbTypeEnum.HIVE == entity.getType()) {
|
||||
if (ProductTypeEnum.HIVE == entity.getType()) {
|
||||
throw new DbswitchException(ResultCode.ERROR_INVALID_ASSIGNMENT_CONFIG, "不支持目的端数据源为Hive");
|
||||
}
|
||||
if (SupportDbTypeEnum.SQLITE3 == entity.getType()) {
|
||||
if (SupportDbTypeEnum.isUnsupportedTargetSqlite(entity.getUrl())) {
|
||||
if (ProductTypeEnum.SQLITE3 == entity.getType()) {
|
||||
if (ProductTypeEnum.isUnsupportedTargetSqlite(entity.getUrl())) {
|
||||
throw new DbswitchException(ResultCode.ERROR_INVALID_ASSIGNMENT_CONFIG,
|
||||
"不支持目的端数据源为远程服务器上的SQLite或内存方式下的SQLite");
|
||||
}
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.service;
|
||||
|
||||
import com.gitee.dbswitch.admin.common.converter.ConverterFactory;
|
||||
import com.gitee.dbswitch.common.converter.ConverterFactory;
|
||||
import com.gitee.dbswitch.admin.common.exception.DbswitchException;
|
||||
import com.gitee.dbswitch.admin.common.response.PageResult;
|
||||
import com.gitee.dbswitch.admin.common.response.Result;
|
||||
@@ -24,13 +24,13 @@ import com.gitee.dbswitch.admin.model.response.DatabaseTypeDetailResponse;
|
||||
import com.gitee.dbswitch.admin.model.response.DatabaseTypeDriverResponse;
|
||||
import com.gitee.dbswitch.admin.model.response.DbConnectionDetailResponse;
|
||||
import com.gitee.dbswitch.admin.model.response.DbConnectionNameResponse;
|
||||
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
|
||||
import com.gitee.dbswitch.admin.util.PageUtils;
|
||||
import com.gitee.dbswitch.common.entity.CloseableDataSource;
|
||||
import com.gitee.dbswitch.common.type.ProductTypeEnum;
|
||||
import com.gitee.dbswitch.common.util.JdbcUrlUtils;
|
||||
import com.gitee.dbswitch.service.MetadataService;
|
||||
import com.gitee.dbswitch.service.DefaultMetadataService;
|
||||
import com.gitee.dbswitch.data.util.DataSourceUtils;
|
||||
import com.gitee.dbswitch.service.DefaultMetadataService;
|
||||
import com.gitee.dbswitch.service.MetadataService;
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
@@ -53,7 +53,7 @@ public class ConnectionService {
|
||||
|
||||
public MetadataService getMetaDataCoreService(DatabaseConnectionEntity dbConn) {
|
||||
String typeName = dbConn.getType().getName().toUpperCase();
|
||||
SupportDbTypeEnum supportDbType = SupportDbTypeEnum.valueOf(typeName);
|
||||
ProductTypeEnum supportDbType = ProductTypeEnum.valueOf(typeName);
|
||||
if (supportDbType.hasAddress()) {
|
||||
for (String pattern : supportDbType.getUrl()) {
|
||||
final Matcher matcher = JdbcUrlUtils.getPattern(pattern).matcher(dbConn.getUrl());
|
||||
@@ -86,7 +86,7 @@ public class ConnectionService {
|
||||
|
||||
public List<DatabaseTypeDetailResponse> getTypes() {
|
||||
List<DatabaseTypeDetailResponse> lists = new ArrayList<>();
|
||||
for (SupportDbTypeEnum type : SupportDbTypeEnum.values()) {
|
||||
for (ProductTypeEnum type : ProductTypeEnum.values()) {
|
||||
DatabaseTypeDetailResponse detail = new DatabaseTypeDetailResponse();
|
||||
detail.setId(type.getId());
|
||||
detail.setType(type.getName().toUpperCase());
|
||||
@@ -99,7 +99,7 @@ public class ConnectionService {
|
||||
return lists;
|
||||
}
|
||||
|
||||
public List<DatabaseTypeDriverResponse> getDrivers(SupportDbTypeEnum dbTypeEnum) {
|
||||
public List<DatabaseTypeDriverResponse> getDrivers(ProductTypeEnum dbTypeEnum) {
|
||||
List<DatabaseTypeDriverResponse> lists = new ArrayList<>();
|
||||
driverLoadService.getDriverVersionWithPath(dbTypeEnum)
|
||||
.forEach(
|
||||
@@ -256,7 +256,7 @@ public class ConnectionService {
|
||||
|
||||
private void validJdbcUrlFormat(DatabaseConnectionEntity conn) {
|
||||
String typeName = conn.getType().getName().toUpperCase();
|
||||
SupportDbTypeEnum supportDbType = SupportDbTypeEnum.valueOf(typeName);
|
||||
ProductTypeEnum supportDbType = ProductTypeEnum.valueOf(typeName);
|
||||
if (!conn.getUrl().startsWith(supportDbType.getUrlPrefix())) {
|
||||
throw new DbswitchException(ResultCode.ERROR_INVALID_JDBC_URL, conn.getUrl());
|
||||
}
|
||||
|
@@ -1,7 +1,7 @@
|
||||
package com.gitee.dbswitch.admin.service;
|
||||
|
||||
import com.gitee.dbswitch.admin.config.DbswitchConfig;
|
||||
import com.gitee.dbswitch.admin.type.SupportDbTypeEnum;
|
||||
import com.gitee.dbswitch.common.type.ProductTypeEnum;
|
||||
import java.io.File;
|
||||
import java.util.EnumMap;
|
||||
import java.util.HashMap;
|
||||
@@ -20,7 +20,7 @@ import org.springframework.stereotype.Service;
|
||||
@Service
|
||||
public class DriverLoadService {
|
||||
|
||||
private Map<SupportDbTypeEnum, Map<String, File>> drivers = new EnumMap<>(SupportDbTypeEnum.class);
|
||||
private Map<ProductTypeEnum, Map<String, File>> drivers = new EnumMap<>(ProductTypeEnum.class);
|
||||
|
||||
@Resource
|
||||
private DbswitchConfig dbswitchConfig;
|
||||
@@ -45,7 +45,7 @@ public class DriverLoadService {
|
||||
"No drivers type found from path:" + driversBasePath);
|
||||
}
|
||||
for (File type : types) {
|
||||
if (!SupportDbTypeEnum.exists(type.getName())) {
|
||||
if (!ProductTypeEnum.exists(type.getName())) {
|
||||
continue;
|
||||
}
|
||||
File[] driverVersions = type.listFiles();
|
||||
@@ -58,7 +58,7 @@ public class DriverLoadService {
|
||||
throw new IllegalArgumentException(
|
||||
"No driver version jar file found from path:" + driverVersion.getAbsolutePath());
|
||||
}
|
||||
SupportDbTypeEnum typeEnum = SupportDbTypeEnum.of(type.getName());
|
||||
ProductTypeEnum typeEnum = ProductTypeEnum.of(type.getName());
|
||||
Map<String, File> versionMap = drivers.computeIfAbsent(typeEnum, k -> new HashMap<>());
|
||||
versionMap.put(driverVersion.getName(), driverVersion);
|
||||
log.info("Load driver for {} ,version:{},path:{}",
|
||||
@@ -67,16 +67,16 @@ public class DriverLoadService {
|
||||
}
|
||||
}
|
||||
|
||||
public List<String> getDriverVersion(SupportDbTypeEnum dbTypeEnum) {
|
||||
public List<String> getDriverVersion(ProductTypeEnum dbTypeEnum) {
|
||||
return Optional.ofNullable(drivers.get(dbTypeEnum)).orElseGet(HashMap::new)
|
||||
.keySet().stream().collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public Map<String, File> getDriverVersionWithPath(SupportDbTypeEnum dbTypeEnum) {
|
||||
public Map<String, File> getDriverVersionWithPath(ProductTypeEnum dbTypeEnum) {
|
||||
return Optional.ofNullable(drivers.get(dbTypeEnum)).orElse(new HashMap<>());
|
||||
}
|
||||
|
||||
public File getVersionDriverFile(SupportDbTypeEnum dbTypeEnum, String driverVersion) {
|
||||
public File getVersionDriverFile(ProductTypeEnum dbTypeEnum, String driverVersion) {
|
||||
return drivers.get(dbTypeEnum).get(driverVersion);
|
||||
}
|
||||
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.service;
|
||||
|
||||
import com.gitee.dbswitch.admin.common.converter.ConverterFactory;
|
||||
import com.gitee.dbswitch.common.converter.ConverterFactory;
|
||||
import com.gitee.dbswitch.admin.common.response.PageResult;
|
||||
import com.gitee.dbswitch.admin.common.response.Result;
|
||||
import com.gitee.dbswitch.admin.common.response.ResultCode;
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.service;
|
||||
|
||||
import com.gitee.dbswitch.admin.common.converter.ConverterFactory;
|
||||
import com.gitee.dbswitch.common.converter.ConverterFactory;
|
||||
import com.gitee.dbswitch.admin.common.response.PageResult;
|
||||
import com.gitee.dbswitch.admin.common.response.Result;
|
||||
import com.gitee.dbswitch.admin.common.response.ResultCode;
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.service;
|
||||
|
||||
import com.gitee.dbswitch.admin.common.converter.ConverterFactory;
|
||||
import com.gitee.dbswitch.common.converter.ConverterFactory;
|
||||
import com.gitee.dbswitch.admin.common.response.Result;
|
||||
import com.gitee.dbswitch.admin.common.response.ResultCode;
|
||||
import com.gitee.dbswitch.admin.controller.converter.SystemUserDetailConverter;
|
||||
|
@@ -1,119 +0,0 @@
|
||||
// Copyright tang. All rights reserved.
|
||||
// https://gitee.com/inrgihc/dbswitch
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license
|
||||
//
|
||||
// Author: tang (inrgihc@126.com)
|
||||
// Date : 2020/1/2
|
||||
// Location: beijing , china
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.type;
|
||||
|
||||
import java.util.Arrays;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum SupportDbTypeEnum {
|
||||
|
||||
MYSQL(1, "mysql", "com.mysql.jdbc.Driver", 3306,
|
||||
"/* ping */ SELECT 1",
|
||||
"jdbc:mysql://",
|
||||
new String[]{"jdbc:mysql://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
MARIADB(2, "mariadb", "org.mariadb.jdbc.Driver", 3306,
|
||||
"SELECT 1",
|
||||
"jdbc:mariadb://",
|
||||
new String[]{"jdbc:mariadb://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
ORACLE(3, "oracle", "oracle.jdbc.driver.OracleDriver", 1521,
|
||||
"SELECT 'Hello' from DUAL",
|
||||
"jdbc:oracle:thin:@",
|
||||
new String[]{"jdbc:oracle:thin:@{host}:{port}:{database}",
|
||||
"jdbc:oracle:thin:@//{host}[:{port}]/{database}"}),
|
||||
SQLSERVER(4, "sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", 1433,
|
||||
"SELECT 1+2 as a",
|
||||
"jdbc:sqlserver://",
|
||||
new String[]{"jdbc:sqlserver://{host}[:{port}][;DatabaseName={database}][;{params}]"}),
|
||||
POSTGRESQL(5, "postgresql", "org.postgresql.Driver", 5432,
|
||||
"SELECT 1",
|
||||
"jdbc:postgresql://",
|
||||
new String[]{"jdbc:postgresql://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
DB2(6, "db2", "com.ibm.db2.jcc.DB2Driver", 50000,
|
||||
"SELECT 1 FROM SYSIBM.SYSDUMMY1",
|
||||
"jdbc:db2://",
|
||||
new String[]{"jdbc:db2://{host}:{port}/{database}[:{params}]"}),
|
||||
DM(7, "dm", "dm.jdbc.driver.DmDriver", 5236,
|
||||
"SELECT 'Hello' from DUAL",
|
||||
"jdbc:dm://",
|
||||
new String[]{"jdbc:dm://{host}:{port}[/{database}][\\?{params}]"}),
|
||||
KINGBASE(8, "kingbase", "com.kingbase8.Driver", 54321,
|
||||
"SELECT 1",
|
||||
"jdbc:kingbase8://",
|
||||
new String[]{"jdbc:kingbase8://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
OSCAR(9, "oscar", "com.oscar.Driver", 2003,
|
||||
"SELECT 1",
|
||||
"jdbc:oscar://",
|
||||
new String[]{"jdbc:oscar://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
GBASE8A(10, "gbase8a", "com.gbase.jdbc.Driver", 5258,
|
||||
"/* ping */ SELECT 1",
|
||||
"jdbc:gbase://",
|
||||
new String[]{"jdbc:gbase://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
SYBASE(11, "sybase", "com.sybase.jdbc4.jdbc.SybDriver", 5000,
|
||||
"SELECT 1+2 as a",
|
||||
"jdbc:sybase:Tds:",
|
||||
new String[]{"jdbc:sybase:Tds:{host}[:{port}][/{database}][\\?{params}]"}),
|
||||
HIVE(12, "hive", "org.apache.hive.jdbc.HiveDriver", 10000,
|
||||
"SELECT 1",
|
||||
"jdbc:hive2://",
|
||||
new String[]{"jdbc:hive2://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
// 参考文章:https://blog.csdn.net/wank1259162/article/details/104946744
|
||||
SQLITE3(13, "sqlite3", "org.sqlite.JDBC", 0,
|
||||
"SELECT 1",
|
||||
"jdbc:sqlite:",
|
||||
new String[]{"jdbc:sqlite:{file}", "jdbc:sqlite::resource:{file}"}),
|
||||
;
|
||||
|
||||
private int id;
|
||||
private String name;
|
||||
private String driver;
|
||||
private int port;
|
||||
private String sql;
|
||||
private String urlPrefix;
|
||||
private String[] url;
|
||||
|
||||
public boolean hasDatabaseName() {
|
||||
return !Arrays.asList(DM, SQLITE3).contains(this);
|
||||
}
|
||||
|
||||
public boolean hasFilePath() {
|
||||
return this == SQLITE3;
|
||||
}
|
||||
|
||||
public boolean hasAddress() {
|
||||
return this != SQLITE3;
|
||||
}
|
||||
|
||||
public static boolean exists(String name) {
|
||||
return Arrays.stream(values()).anyMatch(item -> item.name().equalsIgnoreCase(name));
|
||||
}
|
||||
|
||||
public static SupportDbTypeEnum of(String name) {
|
||||
if (!StringUtils.isEmpty(name)) {
|
||||
for (SupportDbTypeEnum type : SupportDbTypeEnum.values()) {
|
||||
if (type.getName().equalsIgnoreCase(name)) {
|
||||
return type;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new IllegalArgumentException("cannot find enum name: " + name);
|
||||
}
|
||||
|
||||
public static boolean isUnsupportedTargetSqlite(String url) {
|
||||
String prefix1 = "jdbc:sqlite::resource:";
|
||||
//String prefix2 = "jdbc:sqlite::memory:";
|
||||
return url.startsWith(prefix1);
|
||||
}
|
||||
|
||||
}
|
@@ -7,7 +7,7 @@
|
||||
// Date : 2020/1/2
|
||||
// Location: beijing , china
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.common.converter;
|
||||
package com.gitee.dbswitch.common.converter;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
@@ -7,7 +7,7 @@
|
||||
// Date : 2020/1/2
|
||||
// Location: beijing , china
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.common.converter;
|
||||
package com.gitee.dbswitch.common.converter;
|
||||
|
||||
public interface Converter<U, V> {
|
||||
|
@@ -7,7 +7,7 @@
|
||||
// Date : 2020/1/2
|
||||
// Location: beijing , china
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.admin.common.converter;
|
||||
package com.gitee.dbswitch.common.converter;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
@@ -10,104 +10,145 @@
|
||||
package com.gitee.dbswitch.common.type;
|
||||
|
||||
import java.util.Arrays;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Getter;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* 数据库产品类型的枚举定义
|
||||
*
|
||||
* @author Tang
|
||||
*/
|
||||
@Getter
|
||||
@AllArgsConstructor
|
||||
public enum ProductTypeEnum {
|
||||
/**
|
||||
* 未知数据库类型
|
||||
*/
|
||||
UNKNOWN(0, "\""),
|
||||
|
||||
/**
|
||||
* MySQL数据库类型
|
||||
*/
|
||||
MYSQL(1, "`"),
|
||||
|
||||
/**
|
||||
* Oracle数据库类型
|
||||
*/
|
||||
ORACLE(2, "\""),
|
||||
|
||||
/**
|
||||
* SQLServer 2000数据库类型
|
||||
*/
|
||||
SQLSERVER2000(3, "\""),
|
||||
|
||||
/**
|
||||
* SQLServer数据库类型
|
||||
*/
|
||||
SQLSERVER(4, "\""),
|
||||
|
||||
/**
|
||||
* PostgreSQL数据库类型
|
||||
*/
|
||||
POSTGRESQL(5, "\""),
|
||||
|
||||
/**
|
||||
* Greenplum数据库类型
|
||||
*/
|
||||
GREENPLUM(6, "\""),
|
||||
MYSQL(1, "`", "mysql", "com.mysql.jdbc.Driver", 3306,
|
||||
"/* ping */ SELECT 1",
|
||||
"jdbc:mysql://",
|
||||
new String[]{"jdbc:mysql://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
|
||||
/**
|
||||
* MariaDB数据库类型
|
||||
*/
|
||||
MARIADB(7, "`"),
|
||||
MARIADB(2, "`", "mariadb", "org.mariadb.jdbc.Driver", 3306,
|
||||
"SELECT 1",
|
||||
"jdbc:mariadb://",
|
||||
new String[]{"jdbc:mariadb://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
|
||||
/**
|
||||
* Oracle数据库类型
|
||||
*/
|
||||
ORACLE(3, "\"", "oracle", "oracle.jdbc.driver.OracleDriver", 1521,
|
||||
"SELECT 'Hello' from DUAL",
|
||||
"jdbc:oracle:thin:@",
|
||||
new String[]{"jdbc:oracle:thin:@{host}:{port}:{database}",
|
||||
"jdbc:oracle:thin:@//{host}[:{port}]/{database}"}),
|
||||
|
||||
/**
|
||||
* Microsoft SQL Server数据库类型(>=2005)
|
||||
*/
|
||||
SQLSERVER(4, "\"", "sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", 1433,
|
||||
"SELECT 1+2 as a",
|
||||
"jdbc:sqlserver://",
|
||||
new String[]{"jdbc:sqlserver://{host}[:{port}][;DatabaseName={database}][;{params}]"}),
|
||||
|
||||
/**
|
||||
* PostgreSQL数据库类型
|
||||
*/
|
||||
POSTGRESQL(5, "\"", "postgresql", "org.postgresql.Driver", 5432,
|
||||
"SELECT 1",
|
||||
"jdbc:postgresql://",
|
||||
new String[]{"jdbc:postgresql://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
|
||||
/**
|
||||
* DB2数据库类型
|
||||
*/
|
||||
DB2(8, "\""),
|
||||
DB2(6, "\"", "db2", "com.ibm.db2.jcc.DB2Driver", 50000,
|
||||
"SELECT 1 FROM SYSIBM.SYSDUMMY1",
|
||||
"jdbc:db2://",
|
||||
new String[]{"jdbc:db2://{host}:{port}/{database}[:{params}]"}),
|
||||
|
||||
/**
|
||||
* [国产]达梦数据库类型
|
||||
* [国产] 达梦(DM)数据库类型
|
||||
*/
|
||||
DM(9, "\""),
|
||||
DM(7, "\"", "dm", "dm.jdbc.driver.DmDriver", 5236,
|
||||
"SELECT 'Hello' from DUAL",
|
||||
"jdbc:dm://",
|
||||
new String[]{"jdbc:dm://{host}:{port}[/{database}][\\?{params}]"}),
|
||||
|
||||
/**
|
||||
* [国产]人大金仓数据库类型
|
||||
* [国产] 金仓(Kingbase)数据库类型
|
||||
*/
|
||||
KINGBASE(10, "\""),
|
||||
KINGBASE(8, "\"", "kingbase", "com.kingbase8.Driver", 54321,
|
||||
"SELECT 1",
|
||||
"jdbc:kingbase8://",
|
||||
new String[]{"jdbc:kingbase8://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
|
||||
/**
|
||||
* [国产]神通数据库
|
||||
* [国产] 神通(Oscar)数据库类型
|
||||
*/
|
||||
OSCAR(11, "\""),
|
||||
OSCAR(9, "\"", "oscar", "com.oscar.Driver", 2003,
|
||||
"SELECT 1",
|
||||
"jdbc:oscar://",
|
||||
new String[]{"jdbc:oscar://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
|
||||
/**
|
||||
* [国产]南大通用GBase8a数据库
|
||||
* [国产] 南大通用(GBase8A)数据库类型
|
||||
*/
|
||||
GBASE8A(12, "`"),
|
||||
GBASE8A(10, "`", "gbase8a", "com.gbase.jdbc.Driver", 5258,
|
||||
"/* ping */ SELECT 1",
|
||||
"jdbc:gbase://",
|
||||
new String[]{"jdbc:gbase://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
|
||||
/**
|
||||
* HIVE数据库
|
||||
* Sybase 数据库类型
|
||||
*/
|
||||
HIVE(13, "`"),
|
||||
SYBASE(11, "\"", "sybase", "com.sybase.jdbc4.jdbc.SybDriver", 5000,
|
||||
"SELECT 1+2 as a",
|
||||
"jdbc:sybase:Tds:",
|
||||
new String[]{"jdbc:sybase:Tds:{host}[:{port}][/{database}][\\?{params}]"}),
|
||||
|
||||
/**
|
||||
* SQLite数据库
|
||||
* Hive 数据库类型
|
||||
*/
|
||||
SQLITE3(14, "\""),
|
||||
HIVE(12, "`", "hive", "org.apache.hive.jdbc.HiveDriver", 10000,
|
||||
"SELECT 1",
|
||||
"jdbc:hive2://",
|
||||
new String[]{"jdbc:hive2://{host}[:{port}]/[{database}][\\?{params}]"}),
|
||||
|
||||
/**
|
||||
* Sybase数据库类型
|
||||
* Sqlite v3数据库类型
|
||||
*/
|
||||
SYBASE(15, "\""),
|
||||
// 参考文章:https://blog.csdn.net/wank1259162/article/details/104946744
|
||||
SQLITE3(13, "\"", "sqlite3", "org.sqlite.JDBC", 0,
|
||||
"SELECT 1",
|
||||
"jdbc:sqlite:",
|
||||
new String[]{"jdbc:sqlite:{file}", "jdbc:sqlite::resource:{file}"}),
|
||||
;
|
||||
|
||||
private int index;
|
||||
private int id;
|
||||
private String quote;
|
||||
private String name;
|
||||
private String driver;
|
||||
private int port;
|
||||
private String sql;
|
||||
private String urlPrefix;
|
||||
private String[] url;
|
||||
|
||||
ProductTypeEnum(int idx, String quote) {
|
||||
this.index = idx;
|
||||
this.quote = quote;
|
||||
public boolean hasDatabaseName() {
|
||||
return !Arrays.asList(DM, SQLITE3).contains(this);
|
||||
}
|
||||
|
||||
public int getIndex() {
|
||||
return index;
|
||||
public boolean hasFilePath() {
|
||||
return this == SQLITE3;
|
||||
}
|
||||
|
||||
public boolean hasAddress() {
|
||||
return this != SQLITE3;
|
||||
}
|
||||
|
||||
public boolean noCommentStatement() {
|
||||
@@ -129,4 +170,26 @@ public enum ProductTypeEnum {
|
||||
return String.format("%s%s%s.%s%s%s", quote, schema, quote, quote, table, quote);
|
||||
}
|
||||
|
||||
public static boolean exists(String name) {
|
||||
return Arrays.stream(values()).anyMatch(item -> item.name().equalsIgnoreCase(name));
|
||||
}
|
||||
|
||||
public static ProductTypeEnum of(String name) {
|
||||
if (!StringUtils.isEmpty(name)) {
|
||||
for (ProductTypeEnum type : ProductTypeEnum.values()) {
|
||||
if (type.getName().equalsIgnoreCase(name)) {
|
||||
return type;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new IllegalArgumentException("cannot find enum name: " + name);
|
||||
}
|
||||
|
||||
public static boolean isUnsupportedTargetSqlite(String url) {
|
||||
String prefix1 = "jdbc:sqlite::resource:";
|
||||
//String prefix2 = "jdbc:sqlite::memory:";
|
||||
return url.startsWith(prefix1);
|
||||
}
|
||||
|
||||
}
|
||||
|
@@ -35,7 +35,6 @@ public final class DatabaseAwareUtils {
|
||||
productNameMap = new HashMap<>();
|
||||
driverNameMap = new HashMap<>();
|
||||
|
||||
productNameMap.put("Greenplum", ProductTypeEnum.GREENPLUM);
|
||||
productNameMap.put("Microsoft SQL Server", ProductTypeEnum.SQLSERVER);
|
||||
productNameMap.put("DM DBMS", ProductTypeEnum.DM);
|
||||
productNameMap.put("KingbaseES", ProductTypeEnum.KINGBASE);
|
||||
@@ -116,5 +115,5 @@ public final class DatabaseAwareUtils {
|
||||
throw new RuntimeException(se);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
@@ -9,6 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.common.util;
|
||||
|
||||
import cn.hutool.core.convert.Convert;
|
||||
import java.lang.reflect.Field;
|
||||
import java.sql.Types;
|
||||
import java.util.HashMap;
|
||||
@@ -134,5 +135,20 @@ public final class JdbcTypesUtils {
|
||||
// REF
|
||||
// DATALINK
|
||||
// REF_CURSOR
|
||||
|
||||
|
||||
public static long getObjectSize(int jdbcType, Object value) {
|
||||
if (null == value) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (isBinary(jdbcType)) {
|
||||
byte[] bytes = Convert.toPrimitiveByteArray(value);
|
||||
return null == bytes ? 0 : bytes.length;
|
||||
} else if (isBoolean(jdbcType)) {
|
||||
return 1;
|
||||
} else {
|
||||
String strValue = Convert.toStr(value);
|
||||
return null == strValue ? 0 : strValue.length();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@@ -15,8 +15,8 @@ import com.gitee.dbswitch.common.util.JdbcTypesUtils;
|
||||
import com.gitee.dbswitch.common.util.TypeConvertUtils;
|
||||
import com.gitee.dbswitch.provider.ProductProviderFactory;
|
||||
import com.gitee.dbswitch.provider.query.TableDataQueryProvider;
|
||||
import com.gitee.dbswitch.service.MetadataService;
|
||||
import com.gitee.dbswitch.service.DefaultMetadataService;
|
||||
import com.gitee.dbswitch.service.MetadataService;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.sql.SQLException;
|
||||
@@ -268,6 +268,7 @@ public final class ChangeCalculatorService implements IDatabaseChangeCalculator
|
||||
}
|
||||
|
||||
// 初始化计算结果数据字段列信息
|
||||
int[] jdbcTypes = new int[metaData.getColumnCount()];
|
||||
List<String> targetColumns = new ArrayList<>();
|
||||
for (int k = 1; k <= metaData.getColumnCount(); ++k) {
|
||||
String key = metaData.getColumnLabel(k);
|
||||
@@ -275,6 +276,7 @@ public final class ChangeCalculatorService implements IDatabaseChangeCalculator
|
||||
key = metaData.getColumnName(k);
|
||||
}
|
||||
targetColumns.add(columnsMap.getOrDefault(key, key));
|
||||
jdbcTypes[k - 1] = metaData.getColumnType(k);
|
||||
}
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
@@ -329,7 +331,7 @@ public final class ChangeCalculatorService implements IDatabaseChangeCalculator
|
||||
}
|
||||
|
||||
// 这里对计算的单条记录结果进行处理
|
||||
handler.handle(Collections.unmodifiableList(targetColumns), outputRow, flagField);
|
||||
handler.handle(Collections.unmodifiableList(targetColumns), outputRow, jdbcTypes, flagField);
|
||||
}
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
|
@@ -21,11 +21,12 @@ public interface IDatabaseRowHandler {
|
||||
/**
|
||||
* 行数据处理
|
||||
*
|
||||
* @param fields 字段名称列表,该列表只读
|
||||
* @param record 一条数据记实录
|
||||
* @param flag 数据变化状态
|
||||
* @param fields 字段名称列表,该列表只读
|
||||
* @param record 一条数据记实录
|
||||
* @param jdbcTypes jdbc类型
|
||||
* @param flag 数据变化状态
|
||||
*/
|
||||
void handle(List<String> fields, Object[] record, RecordChangeTypeEnum flag);
|
||||
void handle(List<String> fields, Object[] record, int[] jdbcTypes, RecordChangeTypeEnum flag);
|
||||
|
||||
/**
|
||||
* 计算结束通知
|
||||
|
@@ -350,7 +350,6 @@ public class ColumnMetaData {
|
||||
|
||||
// If we're dealing with PostgreSQL and double precision types
|
||||
if ((desc.getProductType() == ProductTypeEnum.POSTGRESQL
|
||||
|| desc.getProductType() == ProductTypeEnum.GREENPLUM
|
||||
|| desc.getProductType() == ProductTypeEnum.KINGBASE)
|
||||
&& type == java.sql.Types.DOUBLE
|
||||
&& precision >= 16
|
||||
@@ -402,8 +401,7 @@ public class ColumnMetaData {
|
||||
}
|
||||
|
||||
if (desc.getProductType() == ProductTypeEnum.POSTGRESQL
|
||||
|| desc.getProductType() == ProductTypeEnum.KINGBASE
|
||||
|| desc.getProductType() == ProductTypeEnum.GREENPLUM) {
|
||||
|| desc.getProductType() == ProductTypeEnum.KINGBASE) {
|
||||
// undefined size => arbitrary precision
|
||||
if (type == java.sql.Types.NUMERIC && length == 0 && precision == 0) {
|
||||
valtype = ColumnMetaData.TYPE_BIGNUMBER;
|
||||
|
@@ -35,11 +35,6 @@
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.ehcache</groupId>
|
||||
<artifactId>sizeof</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-configuration-processor</artifactId>
|
||||
|
@@ -9,7 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.data.domain;
|
||||
|
||||
import com.gitee.dbswitch.data.util.BytesUnitUtils;
|
||||
import cn.hutool.core.io.unit.DataSizeUtil;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
|
||||
@@ -32,6 +32,6 @@ public class PerfStat {
|
||||
return "Data Source Index: \t" + index + "\n" +
|
||||
"Total Tables Count: \t" + total + "\n" +
|
||||
"Failure Tables count: \t" + failure + "\n" +
|
||||
"Total Transfer Size: \t" + BytesUnitUtils.bytesSizeToHuman(bytes) + "\n";
|
||||
"Total Transfer Size: \t" + DataSizeUtil.format(bytes) + "\n";
|
||||
}
|
||||
}
|
||||
|
@@ -9,6 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.data.handler;
|
||||
|
||||
import cn.hutool.core.io.unit.DataSizeUtil;
|
||||
import com.gitee.dbswitch.calculate.ChangeCalculatorService;
|
||||
import com.gitee.dbswitch.calculate.IDatabaseChangeCalculator;
|
||||
import com.gitee.dbswitch.calculate.IDatabaseRowHandler;
|
||||
@@ -18,10 +19,10 @@ import com.gitee.dbswitch.common.entity.CloseableDataSource;
|
||||
import com.gitee.dbswitch.common.entity.ResultSetWrapper;
|
||||
import com.gitee.dbswitch.common.type.ProductTypeEnum;
|
||||
import com.gitee.dbswitch.common.util.DatabaseAwareUtils;
|
||||
import com.gitee.dbswitch.common.util.JdbcTypesUtils;
|
||||
import com.gitee.dbswitch.common.util.PatterNameUtils;
|
||||
import com.gitee.dbswitch.data.config.DbswichProperties;
|
||||
import com.gitee.dbswitch.data.entity.SourceDataSourceProperties;
|
||||
import com.gitee.dbswitch.data.util.BytesUnitUtils;
|
||||
import com.gitee.dbswitch.provider.ProductFactoryProvider;
|
||||
import com.gitee.dbswitch.provider.ProductProviderFactory;
|
||||
import com.gitee.dbswitch.provider.meta.MetadataProvider;
|
||||
@@ -31,9 +32,10 @@ import com.gitee.dbswitch.provider.sync.TableDataSynchronizer;
|
||||
import com.gitee.dbswitch.provider.write.TableDataWriteProvider;
|
||||
import com.gitee.dbswitch.schema.ColumnDescription;
|
||||
import com.gitee.dbswitch.schema.TableDescription;
|
||||
import com.gitee.dbswitch.service.MetadataService;
|
||||
import com.gitee.dbswitch.service.DefaultMetadataService;
|
||||
import com.gitee.dbswitch.service.MetadataService;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
@@ -45,7 +47,6 @@ import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.function.Supplier;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.ehcache.sizeof.SizeOf;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
@@ -57,7 +58,7 @@ import org.springframework.util.StringUtils;
|
||||
@Slf4j
|
||||
public class MigrationHandler implements Supplier<Long> {
|
||||
|
||||
private final long MAX_CACHE_BYTES_SIZE = 512 * 1024 * 1024;
|
||||
private final long MAX_CACHE_BYTES_SIZE = 128 * 1024 * 1024;
|
||||
|
||||
private int fetchSize = 100;
|
||||
private final DbswichProperties properties;
|
||||
@@ -360,15 +361,19 @@ public class MigrationHandler implements Supplier<Long> {
|
||||
long totalCount = 0;
|
||||
long totalBytes = 0;
|
||||
try (ResultSet rs = srs.getResultSet()) {
|
||||
ResultSetMetaData metaData = rs.getMetaData();
|
||||
while (rs.next()) {
|
||||
if (interrupted) {
|
||||
log.info("task job is interrupted!");
|
||||
throw new RuntimeException("task is interrupted");
|
||||
}
|
||||
Object[] record = new Object[sourceFields.size()];
|
||||
long bytes = 0;
|
||||
for (int i = 1; i <= sourceFields.size(); ++i) {
|
||||
try {
|
||||
record[i - 1] = rs.getObject(i);
|
||||
Object value = rs.getObject(i);
|
||||
bytes += JdbcTypesUtils.getObjectSize(metaData.getColumnType(i), value);
|
||||
record[i - 1] = value;
|
||||
} catch (Exception e) {
|
||||
log.warn("!!! Read data from table [ {} ] use function ResultSet.getObject() error",
|
||||
tableNameMapString, e);
|
||||
@@ -377,14 +382,13 @@ public class MigrationHandler implements Supplier<Long> {
|
||||
}
|
||||
|
||||
cache.add(record);
|
||||
long bytes = SizeOf.newInstance().deepSizeOf(record);
|
||||
cacheBytes += bytes;
|
||||
++totalCount;
|
||||
|
||||
if (cache.size() >= BATCH_SIZE || cacheBytes >= MAX_CACHE_BYTES_SIZE) {
|
||||
long ret = writer.write(targetFields, cache);
|
||||
log.info("[FullCoverSync] handle table [{}] data count: {}, the batch bytes sie: {}",
|
||||
tableNameMapString, ret, BytesUnitUtils.bytesSizeToHuman(cacheBytes));
|
||||
tableNameMapString, ret, DataSizeUtil.format(cacheBytes));
|
||||
cache.clear();
|
||||
totalBytes += cacheBytes;
|
||||
cacheBytes = 0;
|
||||
@@ -394,13 +398,13 @@ public class MigrationHandler implements Supplier<Long> {
|
||||
if (cache.size() > 0) {
|
||||
long ret = writer.write(targetFields, cache);
|
||||
log.info("[FullCoverSync] handle table [{}] data count: {}, last batch bytes sie: {}",
|
||||
tableNameMapString, ret, BytesUnitUtils.bytesSizeToHuman(cacheBytes));
|
||||
tableNameMapString, ret, DataSizeUtil.format(cacheBytes));
|
||||
cache.clear();
|
||||
totalBytes += cacheBytes;
|
||||
}
|
||||
|
||||
log.info("[FullCoverSync] handle table [{}] total data count:{}, total bytes={}",
|
||||
tableNameMapString, totalCount, BytesUnitUtils.bytesSizeToHuman(totalBytes));
|
||||
tableNameMapString, totalCount, DataSizeUtil.format(totalBytes));
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
} finally {
|
||||
@@ -465,7 +469,7 @@ public class MigrationHandler implements Supplier<Long> {
|
||||
private final List<Object[]> cacheDelete = new LinkedList<>();
|
||||
|
||||
@Override
|
||||
public void handle(List<String> fields, Object[] record, RecordChangeTypeEnum flag) {
|
||||
public void handle(List<String> fields, Object[] record, int[] jdbcTypes, RecordChangeTypeEnum flag) {
|
||||
if (flag == RecordChangeTypeEnum.VALUE_INSERT) {
|
||||
cacheInsert.add(record);
|
||||
countInsert++;
|
||||
@@ -477,7 +481,11 @@ public class MigrationHandler implements Supplier<Long> {
|
||||
countDelete++;
|
||||
}
|
||||
|
||||
long bytes = SizeOf.newInstance().deepSizeOf(record);
|
||||
long bytes = 0;
|
||||
for (int i = 0; i < record.length; ++i) {
|
||||
Object value = record[i];
|
||||
bytes += JdbcTypesUtils.getObjectSize(jdbcTypes[i], value);
|
||||
}
|
||||
cacheBytes += bytes;
|
||||
totalBytes.addAndGet(bytes);
|
||||
countTotal++;
|
||||
@@ -509,7 +517,7 @@ public class MigrationHandler implements Supplier<Long> {
|
||||
}
|
||||
|
||||
log.info("[IncreaseSync] Handle table [{}] data one batch size: {}",
|
||||
tableNameMapString, BytesUnitUtils.bytesSizeToHuman(cacheBytes));
|
||||
tableNameMapString, DataSizeUtil.format(cacheBytes));
|
||||
cacheBytes = 0;
|
||||
}
|
||||
}
|
||||
|
@@ -9,6 +9,7 @@
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.data.service;
|
||||
|
||||
import cn.hutool.core.io.unit.DataSizeUtil;
|
||||
import cn.hutool.core.stream.StreamUtil;
|
||||
import cn.hutool.core.text.StrPool;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
@@ -21,12 +22,11 @@ import com.gitee.dbswitch.data.config.DbswichProperties;
|
||||
import com.gitee.dbswitch.data.domain.PerfStat;
|
||||
import com.gitee.dbswitch.data.entity.SourceDataSourceProperties;
|
||||
import com.gitee.dbswitch.data.handler.MigrationHandler;
|
||||
import com.gitee.dbswitch.data.util.BytesUnitUtils;
|
||||
import com.gitee.dbswitch.data.util.DataSourceUtils;
|
||||
import com.gitee.dbswitch.data.util.JsonUtils;
|
||||
import com.gitee.dbswitch.schema.TableDescription;
|
||||
import com.gitee.dbswitch.service.MetadataService;
|
||||
import com.gitee.dbswitch.service.DefaultMetadataService;
|
||||
import com.gitee.dbswitch.service.MetadataService;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
@@ -222,7 +222,7 @@ public class MigrationService {
|
||||
log.info(
|
||||
"#### Complete data migration for the [ {} ] data source:\ntotal count={}\nfailure count={}\ntotal bytes size={}",
|
||||
sourcePropertiesIndex, futures.size(), numberOfFailures.get(),
|
||||
BytesUnitUtils.bytesSizeToHuman(totalBytesSize.get()));
|
||||
DataSizeUtil.format(totalBytesSize.get()));
|
||||
perfStats.add(new PerfStat(sourcePropertiesIndex, futures.size(),
|
||||
numberOfFailures.get(), totalBytesSize.get()));
|
||||
++sourcePropertiesIndex;
|
||||
|
@@ -1,51 +0,0 @@
|
||||
// Copyright tang. All rights reserved.
|
||||
// https://gitee.com/inrgihc/dbswitch
|
||||
//
|
||||
// Use of this source code is governed by a BSD-style license
|
||||
//
|
||||
// Author: tang (inrgihc@126.com)
|
||||
// Date : 2020/1/2
|
||||
// Location: beijing , china
|
||||
/////////////////////////////////////////////////////////////
|
||||
package com.gitee.dbswitch.data.util;
|
||||
|
||||
import java.text.DecimalFormat;
|
||||
import lombok.experimental.UtilityClass;
|
||||
|
||||
/**
|
||||
* 字节单位转换
|
||||
*
|
||||
* @author tang
|
||||
*/
|
||||
@UtilityClass
|
||||
public final class BytesUnitUtils {
|
||||
|
||||
public static String bytesSizeToHuman(long size) {
|
||||
/** 定义GB的计算常量 */
|
||||
long GB = 1024 * 1024 * 1024;
|
||||
/** 定义MB的计算常量 */
|
||||
long MB = 1024 * 1024;
|
||||
/** 定义KB的计算常量 */
|
||||
long KB = 1024;
|
||||
|
||||
/** 格式化小数 */
|
||||
DecimalFormat df = new DecimalFormat("0.00");
|
||||
String resultSize = "0.00";
|
||||
|
||||
if (size / GB >= 1) {
|
||||
//如果当前Byte的值大于等于1GB
|
||||
resultSize = df.format(size / (float) GB) + "GB ";
|
||||
} else if (size / MB >= 1) {
|
||||
//如果当前Byte的值大于等于1MB
|
||||
resultSize = df.format(size / (float) MB) + "MB ";
|
||||
} else if (size / KB >= 1) {
|
||||
//如果当前Byte的值大于等于1KB
|
||||
resultSize = df.format(size / (float) KB) + "KB ";
|
||||
} else {
|
||||
resultSize = size + "B ";
|
||||
}
|
||||
|
||||
return resultSize;
|
||||
}
|
||||
|
||||
}
|
Reference in New Issue
Block a user