代码整理

build-docker/dbswitch/Dockerfile

@@ -3,15 +3,6 @@ FROM openjdk:8-jre-alpine
 ENV TZ=Asia/Shanghai
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-ENV TZ=Asia/Shanghai
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-
-ENV TZ=Asia/Shanghai
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-
-ENV TZ=Asia/Shanghai
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-
 ADD dbswitch-release.tar.gz  /
 
 EXPOSE      9088

dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/AssignmentService.java

@@ -41,8 +41,11 @@ import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import javax.annotation.Resource;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.quartz.CronExpression;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
+import org.quartz.CronExpression;
 
 @Service
 public class AssignmentService {

dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/service/PatternMapperService.java

@@ -22,6 +22,9 @@ import org.springframework.stereotype.Service;
 @Service
 public class PatternMapperService {
 
+  private final String STRING_EMPTY = "<!空>";
+  private final String STRING_DELETE = "<!删除>";
+
   @Resource
   private DbConnectionService connectionService;
 
@@ -34,18 +37,22 @@ public class PatternMapperService {
     List<PreviewNameMapperResponse> result = new ArrayList<>();
     if (CollectionUtils.isEmpty(request.getTableNames())) {
       for (TableDescription td : getAllTableNames(request)) {
+        String targetName = PatterNameUtils.getFinalName(
+            td.getTableName(), request.getNameMapper());
         result.add(PreviewNameMapperResponse.builder()
             .originalName(td.getTableName())
-            .targetName(PatterNameUtils.getFinalName(td.getTableName(), request.getNameMapper()))
+            .targetName(StringUtils.isNotBlank(targetName) ? targetName : STRING_EMPTY)
             .build());
       }
     } else {
       if (include) {
         for (String name : request.getTableNames()) {
           if (StringUtils.isNotBlank(name)) {
+            String targetName = PatterNameUtils.getFinalName(
+                name, request.getNameMapper());
             result.add(PreviewNameMapperResponse.builder()
                 .originalName(name)
-                .targetName(PatterNameUtils.getFinalName(name, request.getNameMapper()))
+                .targetName(StringUtils.isNotBlank(targetName) ? targetName : STRING_EMPTY)
                 .build());
           }
         }
@@ -92,7 +99,7 @@ public class PatternMapperService {
       } else {
         result.add(PreviewNameMapperResponse.builder()
             .originalName(cd.getFieldName())
-            .targetName("<!字段被删除>")
+            .targetName(STRING_DELETE)
             .build());
       }
     }
@@ -110,14 +117,10 @@ public class PatternMapperService {
     if (null == dbConn) {
       throw new DbswitchException(ResultCode.ERROR_RESOURCE_NOT_EXISTS, "id=" + request.getId());
     }
+    
     IMetaDataByJdbcService service = connectionService.getMetaDataCoreService(dbConn);
-    return service.queryTableList(
+    return service.queryTableList(dbConn.getUrl(), dbConn.getUsername(), dbConn.getPassword(),
-        dbConn.getUrl(),
+        request.getSchemaName()).stream().filter(td -> !td.isViewTable())
-        dbConn.getUsername(),
-        dbConn.getPassword(),
-        request.getSchemaName()
-    ).stream()
-        .filter(td -> !td.isViewTable())
         .collect(Collectors.toList());
   }
 

dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/AbstractDatabase.java

@@ -9,45 +9,40 @@
 /////////////////////////////////////////////////////////////
 package com.gitee.dbswitch.core.database;
 
-import java.util.List;
+import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
-import java.util.Objects;
+import com.gitee.dbswitch.common.util.DbswitchStrUtils;
-import java.util.Properties;
+import com.gitee.dbswitch.common.util.HivePrepareUtils;
-import java.util.Set;
+import com.gitee.dbswitch.common.util.TypeConvertUtils;
-import java.util.ArrayList;
+import com.gitee.dbswitch.core.model.ColumnDescription;
-import java.util.HashSet;
+import com.gitee.dbswitch.core.model.ColumnMetaData;
+import com.gitee.dbswitch.core.model.SchemaTableData;
+import com.gitee.dbswitch.core.model.TableDescription;
 import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.DriverManager;
-import java.sql.Statement;
-import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 import org.apache.commons.lang3.StringUtils;
-import com.gitee.dbswitch.common.constant.DatabaseTypeEnum;
-import com.gitee.dbswitch.core.model.ColumnDescription;
-import com.gitee.dbswitch.core.model.ColumnMetaData;
-import com.gitee.dbswitch.core.model.TableDescription;
-import com.gitee.dbswitch.core.util.JdbcOperatorUtils;
 
 /**
  * 数据库元信息抽象基类
  *
  * @author tang
- *
  */
 public abstract class AbstractDatabase implements IDatabaseInterface {
 
   public static final int CLOB_LENGTH = 9999999;
 
-	protected Connection connection = null;
+  protected String driverClassName;
-	protected DatabaseMetaData metaData = null;
   protected String catalogName = null;
 
   public AbstractDatabase(String driverClassName) {
-		this.catalogName = null;
-
     try {
+      this.driverClassName = driverClassName;
       Class.forName(driverClassName);
     } catch (ClassNotFoundException e) {
       throw new RuntimeException(e);
@@ -55,77 +50,30 @@ public abstract class AbstractDatabase implements IDatabaseInterface {
   }
 
   @Override
-	public void connect(String jdbcUrl, String username, String password) {
+  public String getDriverClassName() {
-		/*
+    return this.driverClassName;
-		 * 超时时间设置问题： https://blog.csdn.net/lsunwing/article/details/79461217
-		 * https://blog.csdn.net/weixin_34405332/article/details/91664781
-		 */
-		try {
-			/**
-			 * Oracle在通过jdbc连接的时候需要添加一个参数来设置是否获取注释
-			 */
-			Properties props = new Properties();
-			props.put("user", username);
-			props.put("password", password);
-			props.put("remarksReporting", "true");
-
-			// 设置最大时间
-			DriverManager.setLoginTimeout(15);
-			
-			this.connection = DriverManager.getConnection(jdbcUrl, props);
-			if (Objects.isNull(this.connection)) {
-				throw new RuntimeException("数据库连接失败，连接参数为：" + jdbcUrl);
-			}
-			
-			this.metaData = Objects.requireNonNull(this.connection.getMetaData());
-		} catch (SQLException e) {
-			throw new RuntimeException(e);
-		}
-
   }
 
   @Override
-	public void close() {
+  public List<String> querySchemaList(Connection connection) {
-		if (null != connection) {
-			try {
-				connection.close();
-			} catch (SQLException e) {
-			}
-			connection = null;
-		}
-	}
-
-	@Override
-	public List<String> querySchemaList() {
     Set<String> ret = new HashSet<>();
-		ResultSet schemas = null;
+    try (ResultSet schemas = connection.getMetaData().getSchemas()) {
-		try {
-			schemas = this.metaData.getSchemas();
       while (schemas.next()) {
         ret.add(schemas.getString("TABLE_SCHEM"));
       }
       return new ArrayList<>(ret);
     } catch (SQLException e) {
       throw new RuntimeException(e);
-		} finally {
-			try {
-				if (null != schemas) {
-					schemas.close();
-					schemas = null;
     }
-			} catch (SQLException e) {
-			}
-		}
-
   }
 
   @Override
-	public List<TableDescription> queryTableList(String schemaName) {
+  public List<TableDescription> queryTableList(Connection connection, String schemaName) {
     List<TableDescription> ret = new ArrayList<>();
     Set<String> uniqueSet = new HashSet<>();
-		ResultSet tables = null;
+    String[] types = new String[]{"TABLE", "VIEW"};
-		try {
+    try (ResultSet tables = connection.getMetaData()
-			tables = this.metaData.getTables(this.catalogName, schemaName, "%", new String[] { "TABLE", "VIEW" });
+        .getTables(this.catalogName, schemaName, "%", types)) {
       while (tables.next()) {
         String tableName = tables.getString("TABLE_NAME");
         if (uniqueSet.contains(tableName)) {
@@ -144,56 +92,64 @@ public abstract class AbstractDatabase implements IDatabaseInterface {
       return ret;
     } catch (SQLException e) {
       throw new RuntimeException(e);
-		} finally {
-			try {
-				if (null != tables) {
-					tables.close();
-					tables = null;
-				}
-			} catch (SQLException e) {
-			}
     }
   }
 
   @Override
-	public List<ColumnDescription> queryTableColumnMeta(String schemaName, String tableName) {
+  public TableDescription queryTableMeta(Connection connection, String schemaName,
+      String tableName) {
+    return queryTableList(connection, schemaName).stream()
+        .filter(one -> tableName.equals(one.getTableName()))
+        .findAny().orElse(null);
+  }
+
+  @Override
+  public List<String> queryTableColumnName(Connection connection, String schemaName,
+      String tableName) {
+    Set<String> columns = new HashSet<>();
+    try (ResultSet rs = connection.getMetaData()
+        .getColumns(this.catalogName, schemaName, tableName, null)) {
+      while (rs.next()) {
+        columns.add(rs.getString("COLUMN_NAME"));
+      }
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+    return new ArrayList<>(columns);
+  }
+
+  @Override
+  public List<ColumnDescription> queryTableColumnMeta(Connection connection, String schemaName,
+      String tableName) {
     String sql = this.getTableFieldsQuerySQL(schemaName, tableName);
-		List<ColumnDescription> ret = this.querySelectSqlColumnMeta(sql);
+    List<ColumnDescription> ret = this.querySelectSqlColumnMeta(connection, sql);
-		ResultSet columns = null;
+
-		try {
+    // 补充一下注释信息
-			columns = this.metaData.getColumns(this.catalogName, schemaName, tableName, null);
+    try (ResultSet columns = connection.getMetaData()
+        .getColumns(this.catalogName, schemaName, tableName, null)) {
       while (columns.next()) {
         String columnName = columns.getString("COLUMN_NAME");
         String remarks = columns.getString("REMARKS");
         for (ColumnDescription cd : ret) {
-					if (columnName.equalsIgnoreCase(cd.getFieldName())) {
+          if (columnName.equals(cd.getFieldName())) {
             cd.setRemarks(remarks);
           }
         }
       }
     } catch (SQLException e) {
       throw new RuntimeException(e);
-		} finally {
-			try {
-				if (null != columns) {
-					columns.close();
-					columns = null;
     }
-			} catch (SQLException e) {
-			}
-		}
-
     return ret;
   }
 
   @Override
-	public List<String> queryTablePrimaryKeys(String schemaName, String tableName) {
+  public List<String> queryTablePrimaryKeys(Connection connection, String schemaName,
+      String tableName) {
     Set<String> ret = new HashSet<>();
-		ResultSet primarykeys = null;
+    try (ResultSet primaryKeys = connection.getMetaData()
-		try {
+        .getPrimaryKeys(this.catalogName, schemaName, tableName)) {
-			primarykeys = this.metaData.getPrimaryKeys(this.catalogName, schemaName, tableName);
+      while (primaryKeys.next()) {
-			while (primarykeys.next()) {
+        String name = primaryKeys.getString("COLUMN_NAME");
-				String name = primarykeys.getString("COLUMN_NAME");
         if (!ret.contains(name)) {
           ret.add(name);
         }
@@ -201,24 +157,61 @@ public abstract class AbstractDatabase implements IDatabaseInterface {
       return new ArrayList<>(ret);
     } catch (SQLException e) {
       throw new RuntimeException(e);
-		} finally {
+    }
-			try {
+  }
-				if (null != primarykeys) {
+
-					primarykeys.close();
+  @Override
-					primarykeys = null;
+  public SchemaTableData queryTableData(Connection connection, String schemaName, String tableName,
+      int rowCount) {
+    String fullTableName = getQuotedSchemaTableCombination(schemaName, tableName);
+    String querySQL = String.format("SELECT * FROM %s ", fullTableName);
+    SchemaTableData data = new SchemaTableData();
+    data.setSchemaName(schemaName);
+    data.setTableName(tableName);
+    data.setColumns(new ArrayList<>());
+    data.setRows(new ArrayList<>());
+    try (Statement st = connection.createStatement()) {
+      if (getDatabaseType() == DatabaseTypeEnum.HIVE) {
+        HivePrepareUtils.prepare(connection, schemaName, tableName);
+      }
+
+      try (ResultSet rs = st.executeQuery(querySQL)) {
+        ResultSetMetaData m = rs.getMetaData();
+        int count = m.getColumnCount();
+        for (int i = 1; i <= count; i++) {
+          data.getColumns().add(m.getColumnLabel(i));
+        }
+
+        int counter = 0;
+        while (rs.next() && counter++ < rowCount) {
+          List<Object> row = new ArrayList<>(count);
+          for (int i = 1; i <= count; i++) {
+            Object value = rs.getObject(i);
+            if (value != null && value instanceof byte[]) {
+              row.add(DbswitchStrUtils.toHexString((byte[]) value));
+            } else if (value != null && value instanceof java.sql.Clob) {
+              row.add(TypeConvertUtils.castToString(value));
+            } else if (value != null && value instanceof java.sql.Blob) {
+              byte[] bytes = TypeConvertUtils.castToByteArray(value);
+              row.add(DbswitchStrUtils.toHexString(bytes));
+            } else {
+              row.add(null == value ? null : value.toString());
+            }
+          }
+          data.getRows().add(row);
+        }
+
+        return data;
       }
     } catch (SQLException e) {
-			}
+      throw new RuntimeException(e);
     }
   }
 
   @Override
-	public abstract List<ColumnDescription> querySelectSqlColumnMeta(String sql);
+  public void testQuerySQL(Connection connection, String sql) {
-
-	@Override
-	public void testQuerySQL(String sql) {
     String wrapperSql = this.getTestQuerySQL(sql);
-		try (Statement statement = this.connection.createStatement();) {
+    try (Statement statement = connection.createStatement();) {
       statement.execute(wrapperSql);
     } catch (SQLException e) {
       throw new RuntimeException(e);
@@ -231,8 +224,9 @@ public abstract class AbstractDatabase implements IDatabaseInterface {
   }
 
   @Override
-	public String getFieldDefinition(ColumnMetaData v, List<String> pks, boolean useAutoInc, boolean addCr) {
+  public String getFieldDefinition(ColumnMetaData v, List<String> pks, boolean useAutoInc,
-		throw new RuntimeException("AbstractDatabase Unempliment!");
+      boolean addCr, boolean withRemarks) {
+    throw new RuntimeException("AbstractDatabase Unimplemented!");
   }
 
   @Override
@@ -248,6 +242,12 @@ public abstract class AbstractDatabase implements IDatabaseInterface {
     return "";
   }
 
+  @Override
+  public List<String> getTableColumnCommentDefinition(TableDescription td,
+      List<ColumnDescription> cds) {
+    throw new RuntimeException("AbstractDatabase Unimplemented!");
+  }
+
   /**************************************
    * internal function
    **************************************/
@@ -256,15 +256,14 @@ public abstract class AbstractDatabase implements IDatabaseInterface {
 
   protected abstract String getTestQuerySQL(String sql);
 
-	protected List<ColumnDescription> getSelectSqlColumnMeta(String querySQL, DatabaseTypeEnum dbtype) {
+  protected List<ColumnDescription> getSelectSqlColumnMeta(Connection connection, String querySQL) {
-		List<ColumnDescription> ret = new ArrayList<ColumnDescription>();
+    List<ColumnDescription> ret = new ArrayList<>();
-		PreparedStatement pstmt = null;
+    try (Statement st = connection.createStatement()) {
-		ResultSet rs = null;
+      if (getDatabaseType() == DatabaseTypeEnum.HIVE) {
-
+        HivePrepareUtils.setResultSetColumnNameNotUnique(connection);
-		try {
+      }
-			pstmt = this.connection.prepareStatement(querySQL);
-			rs = pstmt.executeQuery();
 
+      try (ResultSet rs = st.executeQuery(querySQL)) {
         ResultSetMetaData m = rs.getMetaData();
         int columns = m.getColumnCount();
         for (int i = 1; i <= columns; i++) {
@@ -304,17 +303,16 @@ public abstract class AbstractDatabase implements IDatabaseInterface {
             // nothing more we can do here by catch the exception.
           }
           cd.setSigned(signed);
-				cd.setDbType(dbtype);
+          cd.setDbType(getDatabaseType());
 
           ret.add(cd);
         }
 
         return ret;
+      }
     } catch (SQLException e) {
       throw new RuntimeException(e);
-		} finally {
-			JdbcOperatorUtils.closeResultSet(rs);
-			JdbcOperatorUtils.closeStatement(pstmt);
     }
   }
+
 }

dbswitch-data/src/main/java/com/gitee/dbswitch/data/handler/MigrationHandler.java

@@ -34,7 +34,6 @@ import com.gitee.dbswitch.dbwriter.IDatabaseWriter;
 import com.zaxxer.hikari.HikariDataSource;
 import java.sql.ResultSet;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -114,6 +113,10 @@ public class MigrationHandler implements Supplier<Long> {
     this.targetTableName = PatterNameUtils.getFinalName(td.getTableName(),
         sourceProperties.getRegexTableMapper());
 
+    if (StringUtils.isEmpty(this.targetTableName)) {
+      throw new RuntimeException("表名的映射规则配置有误，不能将[" + this.sourceTableName + "]映射为空");
+    }
+
     this.tableNameMapString = String.format("%s.%s --> %s.%s",
         td.getSchemaName(), td.getTableName(),
         targetSchemaName, targetTableName);
@@ -160,16 +163,26 @@ public class MigrationHandler implements Supplier<Long> {
       String targetColumnName = targetColumnDescriptions.get(i).getFieldName();
       if (StringUtils.hasLength(targetColumnName)) {
         columnMapperPairs.add(String.format("%s --> %s", sourceColumnName, targetColumnName));
-      } else {
-        columnMapperPairs.add(String.format("%s --> %s", sourceColumnName, "<!Field is Deleted>"));
-      }
         mapChecker.put(sourceColumnName, targetColumnName);
+      } else {
+        columnMapperPairs.add(String.format(
+            "%s --> %s",
+            sourceColumnName,
+            String.format("<!Field(%s) is Deleted>", (i + 1))
+        ));
+      }
     }
     log.info("Mapping relation : \ntable mapper :\n\t{}  \ncolumn mapper :\n\t{} ",
-        tableNameMapString, columnMapperPairs.stream().collect(Collectors.joining("\n\t")));
+        tableNameMapString, String.join("\n\t", columnMapperPairs));
     Set<String> valueSet = new HashSet<>(mapChecker.values());
+    if (valueSet.size() <= 0) {
+      throw new RuntimeException("字段映射配置有误，禁止通过映射将表所有的字段都删除!");
+    }
+    if (!valueSet.containsAll(this.targetPrimaryKeys)) {
+      throw new RuntimeException("字段映射配置有误，禁止通过映射将表的主键字段删除!");
+    }
     if (mapChecker.keySet().size() != valueSet.size()) {
-      throw new RuntimeException("字段映射配置有误，多个字段映射到一个同名字段!");
+      throw new RuntimeException("字段映射配置有误，禁止将多个字段映射到一个同名字段!");
     }
 
     IDatabaseWriter writer = DatabaseWriterFactory.createDatabaseWriter(
@@ -258,24 +271,16 @@ public class MigrationHandler implements Supplier<Long> {
   private Long doFullCoverSynchronize(IDatabaseWriter writer) {
     final int BATCH_SIZE = fetchSize;
 
-    List<String> sourceFields = sourceColumnDescriptions.stream()
+    List<String> sourceFields = new ArrayList<>();
-        .map(ColumnDescription::getFieldName)
+    List<String> targetFields = new ArrayList<>();
-        .collect(Collectors.toList());
+    for (int i = 0; i < targetColumnDescriptions.size(); ++i) {
-    List<String> targetFields = targetColumnDescriptions.stream()
+      ColumnDescription scd = sourceColumnDescriptions.get(i);
-        .map(ColumnDescription::getFieldName)
+      ColumnDescription tcd = targetColumnDescriptions.get(i);
-        .collect(Collectors.toList());
+      if (!StringUtils.isEmpty(tcd.getFieldName())) {
-    List<Integer> deletedFieldIndexes = new ArrayList<>();
+        sourceFields.add(scd.getFieldName());
-    for (int i = 0; i < targetFields.size(); ++i) {
+        targetFields.add(tcd.getFieldName());
-      if (StringUtils.isEmpty(targetFields.get(i))) {
-        deletedFieldIndexes.add(i);
       }
     }
-    Collections.reverse(deletedFieldIndexes);
-    deletedFieldIndexes.forEach(i -> {
-      sourceFields.remove(sourceFields.get(i));
-      targetFields.remove(targetFields.get(i));
-    });
-
     // 准备目的端的数据写入操作
     writer.prepareWrite(targetSchemaName, targetTableName, targetFields);
 
@@ -351,26 +356,18 @@ public class MigrationHandler implements Supplier<Long> {
    */
   private Long doIncreaseSynchronize(IDatabaseWriter writer) {
     final int BATCH_SIZE = fetchSize;
-    List<String> sourceFields = sourceColumnDescriptions.stream()
+
-        .map(ColumnDescription::getFieldName)
+    List<String> sourceFields = new ArrayList<>();
-        .collect(Collectors.toList());
+    List<String> targetFields = new ArrayList<>();
-    List<String> targetFields = targetColumnDescriptions.stream()
-        .map(ColumnDescription::getFieldName)
-        .collect(Collectors.toList());
-    List<Integer> deletedFieldIndexes = new ArrayList<>();
-    for (int i = 0; i < targetFields.size(); ++i) {
-      if (StringUtils.isEmpty(targetFields.get(i))) {
-        deletedFieldIndexes.add(i);
-      }
-    }
-    Collections.reverse(deletedFieldIndexes);
-    deletedFieldIndexes.forEach(i -> {
-      sourceFields.remove(sourceFields.get(i));
-      targetFields.remove(targetFields.get(i));
-    });
     Map<String, String> columnNameMaps = new HashMap<>();
-    for (int i = 0; i < sourceFields.size(); ++i) {
+    for (int i = 0; i < targetColumnDescriptions.size(); ++i) {
-      columnNameMaps.put(sourceFields.get(i), targetFields.get(i));
+      ColumnDescription scd = sourceColumnDescriptions.get(i);
+      ColumnDescription tcd = targetColumnDescriptions.get(i);
+      if (!StringUtils.isEmpty(tcd.getFieldName())) {
+        sourceFields.add(scd.getFieldName());
+        targetFields.add(tcd.getFieldName());
+        columnNameMaps.put(scd.getFieldName(), tcd.getFieldName());
+      }
     }
 
     TaskParamEntity.TaskParamEntityBuilder taskBuilder = TaskParamEntity.builder();