支持clickhouse

README.md

@@ -45,6 +45,7 @@
         ├── dbswitch-product-sybase   //  -> sybase方言实现类
         ├── dbswitch-product-hive     //  -> hive方言实现类
         ├── dbswitch-product-sqlite   //  -> sqlite方言实现类
+        ├── dbswitch-product-clickhouse//  -> clickhouse方言实现类
         ├── dbswitch-product-mongodb  //  -> mongodb方言实现类
     ├── dbswitch-data                 // 工具入口模块，读取配置文件中的参数执行异构迁移同步
     ├── dbswitch-admin                // 在以上模块的基础上引入Quartz的调度服务与接口
@@ -320,6 +321,13 @@ jdbc连接地址：dbc:opengauss://172.17.2.10:5866/test
 jdbc驱动名称：org.opengauss.Driver
 ```
 
+**ClickHouse数据库**
+
+```
+jdbc连接地址：jdbc:clickhouse://172.17.2.10:8123/default
+jdbc驱动名称：com.clickhouse.jdbc.ClickHouseDriver
+```
+
 **SQLite数据库**
  
 ```

dbswitch-common/src/main/java/com/gitee/dbswitch/common/type/ProductTypeEnum.java

@@ -137,10 +137,18 @@ public enum ProductTypeEnum {
       "jdbc:opengauss://",
       new String[]{"jdbc:opengauss://{host}[:{port}]/[{database}][\\?{params}]"}),
 
+  /**
+   * ClickHouse数据库类型
+   */
+  CLICKHOUSE(15, "`", "clickhouse", "com.clickhouse.jdbc.ClickHouseDriver", 8123,
+      "SELECT 1",
+      "jdbc:clickhouse://",
+      new String[]{"jdbc:clickhouse://{host}[:{port}]/[{database}][\\?{params}]"}),
+
   /**
    * MongoDB数据库类型
    */
-  MONGODB(15, "\"", "mongoDB", "com.gitee.jdbc.mongodb.JdbcDriver", 27017,
+  MONGODB(16, "\"", "mongoDB", "com.gitee.jdbc.mongodb.JdbcDriver", 27017,
       "use admin;",
       "jdbc:mongodb://",
       new String[]{"jdbc:mongodb://{host}[:{port}]/[{database}][\\?{params}]"}),
@@ -240,6 +248,15 @@ public enum ProductTypeEnum {
     return this == MONGODB;
   }
 
+  /**
+   * 是否为ClickHouse数据库类型
+   *
+   * @return boolean
+   */
+  public boolean isClickHouse() {
+    return this == CLICKHOUSE;
+  }
+
   /**
    * 是否存在指定字符串名称的数据库类型
    *

dbswitch-common/src/main/java/com/gitee/dbswitch/common/util/DatabaseAwareUtils.java

@@ -49,6 +49,7 @@ public final class DatabaseAwareUtils {
     productNameMap.put("OSCAR", ProductTypeEnum.OSCAR);
     productNameMap.put("GBase", ProductTypeEnum.GBASE8A);
     productNameMap.put("Adaptive Server Enterprise", ProductTypeEnum.SYBASE);
+    productNameMap.put("ClickHouse", ProductTypeEnum.CLICKHOUSE);
 
     driverNameMap.put("MySQL Connector Java", ProductTypeEnum.MYSQL);
     driverNameMap.put("MariaDB Connector/J", ProductTypeEnum.MARIADB);
@@ -62,6 +63,7 @@ public final class DatabaseAwareUtils {
     driverNameMap.put("OSCAR JDBC DRIVER", ProductTypeEnum.OSCAR);
     driverNameMap.put("GBase JDBC Driver", ProductTypeEnum.GBASE8A);
     driverNameMap.put("jConnect (TM) for JDBC (TM)", ProductTypeEnum.SYBASE);
+    driverNameMap.put("ClickHouse JDBC Driver", ProductTypeEnum.CLICKHOUSE);
   }
 
   /**

dbswitch-common/src/main/java/com/gitee/dbswitch/common/util/JdbcUrlUtils.java

@@ -318,6 +318,16 @@ public final class JdbcUrlUtils {
     } else {
       System.out.println("error for mongodb!");
     }
+
+    // 13、ClickHouse数据库
+    // jdbc:clickhouse://127.0.0.1:8123/default
+    final Matcher matcher12 = JdbcUrlUtils.getPattern("jdbc:clickhouse://{host}[:{port}]/[{database}][\\?{params}]")
+        .matcher("jdbc:clickhouse://127.0.0.1:8123/default");
+    if (matcher12.matches()) {
+      System.out.println("clickhouse database:" + matcher12.group("database"));
+    } else {
+      System.out.println("error for clickhouse!");
+    }
   }
 
 }

dbswitch-core/src/main/java/com/gitee/dbswitch/util/GenerateSqlUtils.java

@@ -118,6 +118,16 @@ public final class GenerateSqlUtils {
         sb.append(Constants.CR);
         sb.append("STORED AS ORC");
       }
+    } else if (type.isClickHouse()) {
+      sb.append("ENGINE=MergeTree");
+      if (CollectionUtils.isEmpty(pks)) {
+        sb.append(Constants.CR);
+        sb.append("ORDER BY tuple()");
+      }
+      if (withRemarks && StringUtils.isNotBlank(tableRemarks)) {
+        sb.append(Constants.CR);
+        sb.append(String.format("COMMENT='%s' ", tableRemarks.replace("'", "\\'")));
+      }
     }
 
     return DDLFormatterUtils.format(sb.toString());

dbswitch-product/dbswitch-product-clickhouse/pom.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>dbswitch-product</artifactId>
+    <groupId>com.gitee.dbswitch</groupId>
+    <version>1.8.2</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>dbswitch-product-clickhouse</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.gitee.dbswitch</groupId>
+      <artifactId>dbswitch-common</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.gitee.dbswitch</groupId>
+      <artifactId>dbswitch-core</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+
+  </dependencies>
+
+</project>

dbswitch-product/dbswitch-product-clickhouse/src/main/java/com/gitee/dbswitch/product/clickhouse/ClickhouseFactoryProvider.java

@@ -0,0 +1,56 @@
+// Copyright tang.  All rights reserved.
+// https://gitee.com/inrgihc/dbswitch
+//
+// Use of this source code is governed by a BSD-style license
+//
+// Author: tang (inrgihc@126.com)
+// Date : 2020/1/2
+// Location: beijing , china
+/////////////////////////////////////////////////////////////
+package com.gitee.dbswitch.product.clickhouse;
+
+import com.gitee.dbswitch.annotation.Product;
+import com.gitee.dbswitch.common.type.ProductTypeEnum;
+import com.gitee.dbswitch.features.ProductFeatures;
+import com.gitee.dbswitch.provider.AbstractFactoryProvider;
+import com.gitee.dbswitch.provider.meta.MetadataProvider;
+import com.gitee.dbswitch.provider.operate.DefaultTableOperateProvider;
+import com.gitee.dbswitch.provider.operate.TableOperateProvider;
+import com.gitee.dbswitch.provider.sync.AutoCastTableDataSynchronizer;
+import com.gitee.dbswitch.provider.sync.TableDataSynchronizer;
+import com.gitee.dbswitch.provider.write.AutoCastTableDataWriteProvider;
+import com.gitee.dbswitch.provider.write.TableDataWriteProvider;
+import javax.sql.DataSource;
+
+@Product(ProductTypeEnum.CLICKHOUSE)
+public class ClickhouseFactoryProvider extends AbstractFactoryProvider {
+
+  public ClickhouseFactoryProvider(DataSource dataSource) {
+    super(dataSource);
+  }
+
+  public ProductFeatures getProductFeatures() {
+    return new ClickhouseFeatures();
+  }
+
+  @Override
+  public MetadataProvider createMetadataQueryProvider() {
+    return new ClickhouseMetadataQueryProvider(this);
+  }
+
+  @Override
+  public TableDataWriteProvider createTableDataWriteProvider(boolean useInsert) {
+    return new AutoCastTableDataWriteProvider(this);
+  }
+
+  @Override
+  public TableOperateProvider createTableOperateProvider() {
+    return new DefaultTableOperateProvider(this);
+  }
+
+  @Override
+  public TableDataSynchronizer createTableDataSynchronizer() {
+    return new AutoCastTableDataSynchronizer(this);
+  }
+
+}

dbswitch-product/dbswitch-product-clickhouse/src/main/java/com/gitee/dbswitch/product/clickhouse/ClickhouseFeatures.java

@@ -0,0 +1,16 @@
+// Copyright tang.  All rights reserved.
+// https://gitee.com/inrgihc/dbswitch
+//
+// Use of this source code is governed by a BSD-style license
+//
+// Author: tang (inrgihc@126.com)
+// Date : 2020/1/2
+// Location: beijing , china
+/////////////////////////////////////////////////////////////
+package com.gitee.dbswitch.product.clickhouse;
+
+import com.gitee.dbswitch.features.ProductFeatures;
+
+public class ClickhouseFeatures implements ProductFeatures {
+
+}

dbswitch-product/dbswitch-product-clickhouse/src/main/java/com/gitee/dbswitch/product/clickhouse/ClickhouseMetadataQueryProvider.java

@@ -0,0 +1,266 @@
+// Copyright tang.  All rights reserved.
+// https://gitee.com/inrgihc/dbswitch
+//
+// Use of this source code is governed by a BSD-style license
+//
+// Author: tang (inrgihc@126.com)
+// Date : 2020/1/2
+// Location: beijing , china
+/////////////////////////////////////////////////////////////
+package com.gitee.dbswitch.product.clickhouse;
+
+import com.gitee.dbswitch.common.consts.Constants;
+import com.gitee.dbswitch.provider.ProductFactoryProvider;
+import com.gitee.dbswitch.provider.meta.AbstractMetadataProvider;
+import com.gitee.dbswitch.schema.ColumnDescription;
+import com.gitee.dbswitch.schema.ColumnMetaData;
+import com.gitee.dbswitch.schema.TableDescription;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+@Slf4j
+public class ClickhouseMetadataQueryProvider extends AbstractMetadataProvider {
+
+  private static final String SHOW_CREATE_TABLE_SQL = "SHOW CREATE TABLE `%s`.`%s` ";
+  private static final String SHOW_CREATE_VIEW_SQL = "SHOW CREATE VIEW `%s`.`%s` ";
+  private static final String QUERY_SCHEMA_LIST_SQL =
+      "SELECT name from `system`.`databases` where engine !='Memory'";
+  private static final String QUERY_TABLE_LIST_SQL =
+      "SELECT database ,name, comment, engine from `system`.`tables` where is_temporary =0 and database = ? ";
+  private static final String QUERY_TABLE_META_SQL =
+      "SELECT database ,name, comment, engine from `system`.`tables` where is_temporary =0 and database = ? and name = ?";
+  private static final String QUERY_PRIMARY_KEY_SQL =
+      "SELECT name from `system`.`columns` where `database` = ? and `table` = ? and is_in_primary_key =1 order by `position` ";
+
+  public ClickhouseMetadataQueryProvider(ProductFactoryProvider factoryProvider) {
+    super(factoryProvider);
+  }
+
+  @Override
+  public List<String> querySchemaList(Connection connection) {
+    List<String> result = new ArrayList<>();
+    try (Statement st = connection.createStatement()) {
+      try (ResultSet rs = st.executeQuery(QUERY_SCHEMA_LIST_SQL)) {
+        while (rs.next()) {
+          result.add(rs.getString(1));
+        }
+        return result;
+      }
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public List<TableDescription> queryTableList(Connection connection, String schemaName) {
+    List<TableDescription> result = new ArrayList<>();
+    try (PreparedStatement ps = connection.prepareStatement(QUERY_TABLE_LIST_SQL)) {
+      ps.setString(1, schemaName);
+      try (ResultSet rs = ps.executeQuery();) {
+        while (rs.next()) {
+          TableDescription td = new TableDescription();
+          td.setSchemaName(rs.getString(1));
+          td.setTableName(rs.getString(2));
+          td.setRemarks(rs.getString(3));
+          String tableType = rs.getString(4);
+          if (tableType.equalsIgnoreCase("VIEW")) {
+            td.setTableType("VIEW");
+          } else {
+            td.setTableType("TABLE");
+          }
+
+          result.add(td);
+        }
+
+        return result;
+      }
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public TableDescription queryTableMeta(Connection connection, String schemaName, String tableName) {
+    try (PreparedStatement ps = connection.prepareStatement(QUERY_TABLE_META_SQL)) {
+      ps.setString(1, schemaName);
+      ps.setString(2, tableName);
+      try (ResultSet rs = ps.executeQuery();) {
+        if (rs.next()) {
+          TableDescription td = new TableDescription();
+          td.setSchemaName(rs.getString(1));
+          td.setTableName(rs.getString(2));
+          td.setRemarks(rs.getString(3));
+          String tableType = rs.getString(4);
+          if (tableType.equalsIgnoreCase("VIEW")) {
+            td.setTableType("VIEW");
+          } else {
+            td.setTableType("TABLE");
+          }
+
+          return td;
+        }
+
+        return null;
+      }
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public List<String> queryTablePrimaryKeys(Connection connection, String schemaName, String tableName) {
+    List<String> result = new ArrayList<>();
+    try (PreparedStatement ps = connection.prepareStatement(QUERY_PRIMARY_KEY_SQL)) {
+      ps.setString(1, schemaName);
+      ps.setString(2, tableName);
+      try (ResultSet rs = ps.executeQuery();) {
+        while (rs.next()) {
+          result.add(rs.getString(1));
+        }
+        return result;
+      }
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public String getTableDDL(Connection connection, String schemaName, String tableName) {
+    List<String> result = new ArrayList<>();
+    try (Statement st = connection.createStatement()) {
+      if (st.execute(String.format(SHOW_CREATE_TABLE_SQL, schemaName, tableName))) {
+        try (ResultSet rs = st.getResultSet()) {
+          if (rs != null) {
+            while (rs.next()) {
+              String value = rs.getString(1);
+              Optional.ofNullable(value).ifPresent(result::add);
+            }
+          }
+        }
+      }
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+
+    return result.stream().findAny().orElse(null);
+  }
+
+  @Override
+  public String getViewDDL(Connection connection, String schemaName, String tableName) {
+    List<String> result = new ArrayList<>();
+    try (Statement st = connection.createStatement()) {
+      if (st.execute(String.format(SHOW_CREATE_VIEW_SQL, schemaName, tableName))) {
+        try (ResultSet rs = st.getResultSet()) {
+          if (rs != null) {
+            while (rs.next()) {
+              String value = rs.getString(1);
+              Optional.ofNullable(value).ifPresent(result::add);
+            }
+          }
+        }
+      }
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+
+    return result.stream().findAny().orElse(null);
+  }
+
+  @Override
+  public List<ColumnDescription> querySelectSqlColumnMeta(Connection connection, String sql) {
+    String querySQL = String.format(" %s LIMIT 0,1", sql.replace(";", ""));
+    return this.getSelectSqlColumnMeta(connection, querySQL);
+  }
+
+  @Override
+  public void testQuerySQL(Connection connection, String sql) {
+    String testQuerySql = String.format("explain %s", sql.replace(";", ""));
+    if (log.isDebugEnabled()) {
+      log.debug("Execute sql :{}", testQuerySql);
+    }
+    try (Statement st = connection.createStatement()) {
+      st.execute(testQuerySql);
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public String getFieldDefinition(ColumnMetaData v, List<String> pks, boolean useAutoInc,
+      boolean addCr, boolean withRemarks) {
+    String fieldname = v.getName();
+    int length = v.getLength();
+    int precision = v.getPrecision();
+    int type = v.getType();
+
+    String retval = " `" + fieldname + "`  ";
+
+    switch (type) {
+      case ColumnMetaData.TYPE_TIMESTAMP:
+      case ColumnMetaData.TYPE_TIME:
+        retval += "DateTime64";
+        break;
+      case ColumnMetaData.TYPE_DATE:
+        retval += "Date";
+        break;
+      case ColumnMetaData.TYPE_BOOLEAN:
+        retval += "Bool";
+        break;
+      case ColumnMetaData.TYPE_NUMBER:
+      case ColumnMetaData.TYPE_INTEGER:
+      case ColumnMetaData.TYPE_BIGNUMBER:
+        // Integer values...
+        if (precision == 0) {
+          retval += "UInt64";
+        } else {
+          // Floating point values...
+          retval += "Float64";
+        }
+        break;
+      case ColumnMetaData.TYPE_STRING:
+        if (length > 0) {
+          if (length == 1) {
+            retval += "FixedString(1)";
+          } else if (length < 4096) {
+            retval += "FixedString(" + length + ")";
+          } else {
+            retval += "String";
+          }
+        } else {
+          retval += "String";
+        }
+        break;
+      case ColumnMetaData.TYPE_BINARY:
+        retval += "String";
+        break;
+      default:
+        retval += " String";
+        break;
+    }
+
+    if (withRemarks && StringUtils.isNotBlank(v.getRemarks())) {
+      retval += String.format(" COMMENT '%s' ", v.getRemarks().replace("'", "\\'"));
+    }
+
+    if (addCr) {
+      retval += Constants.CR;
+    }
+
+    return retval;
+  }
+
+  @Override
+  public List<String> getTableColumnCommentDefinition(TableDescription td, List<ColumnDescription> cds) {
+    return Collections.emptyList();
+  }
+
+}

dbswitch-product/dbswitch-product-clickhouse/src/main/resources/META-INF/services/dbswitch.providers

@@ -0,0 +1 @@
+com.gitee.dbswitch.product.clickhouse.ClickhouseFactoryProvider

dbswitch-product/dbswitch-register-product/pom.xml

@@ -83,6 +83,11 @@
       <artifactId>dbswitch-product-openguass</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.gitee.dbswitch</groupId>
+      <artifactId>dbswitch-product-clickhouse</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.gitee.dbswitch</groupId>
       <artifactId>dbswitch-product-mongodb</artifactId>

dbswitch-product/pom.xml

@@ -28,6 +28,7 @@
     <module>dbswitch-product-openguass</module>
     <module>dbswitch-product-mongodb</module>
     <module>dbswitch-register-product</module>
+    <module>dbswitch-product-clickhouse</module>
   </modules>
 
 </project>