diff --git a/dbswitch-admin/pom.xml b/dbswitch-admin/pom.xml index 773f96dd..87f8b3c3 100644 --- a/dbswitch-admin/pom.xml +++ b/dbswitch-admin/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.3 + 1.6.4 dbswitch-admin @@ -39,6 +39,12 @@ org.springframework.boot spring-boot-starter-web + + + spring-boot-starter-logging + org.springframework.boot + + diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/type/SupportDbTypeEnum.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/type/SupportDbTypeEnum.java index bc105562..52704fea 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/type/SupportDbTypeEnum.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/type/SupportDbTypeEnum.java @@ -34,6 +34,8 @@ public enum SupportDbTypeEnum { new String[]{"jdbc:dm://{host}:{port}[/{database}][\\?{params}]"}), KINGBASE(8, "kingbase", "com.kingbase8.Driver", 54321, "SELECT 1", new String[]{"jdbc:kingbase8://{host}[:{port}]/[{database}][\\?{params}]"}), + HIVE(1, "hive", "org.apache.hive.jdbc.HiveDriver", 10000, "SELECT 1", + new String[]{"jdbc:hive2://{host}[:{port}]/[{database}][\\?{params}]"}), ; private int id; diff --git a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/JDBCURL.java b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/JDBCURL.java index f8698025..5ed7da70 100644 --- a/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/JDBCURL.java +++ b/dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/JDBCURL.java @@ -283,12 +283,32 @@ public class JDBCURL { System.out.println("error for db2!"); } - // 9、SQLite数据库 - // jdbc:sqlite:/tmp/phone.db - final Matcher matcher9 = JDBCURL.getPattern("jdbc:sqlite:{file}") - .matcher("jdbc:sqlite:D:\\Project\\Test\\phone.db"); + // 10、Hive数据库 + // jdbc:hive2://172.17.2.10:10000/test?useUnicode=true&useSSL=false + final Matcher matcher9 = JDBCURL + .getPattern("jdbc:hive2://{host}[:{port}]/[{database}][\\?{params}]") + .matcher("jdbc:hive2://127.0.0.1:10000/default?useUnicode=true&useSSL=false"); if (matcher9.matches()) { - System.out.println("sqlite file:" + matcher9.group("file")); + System.out.println("hive host:" + matcher3.group("host")); + System.out.println("hive port:" + matcher3.group("port")); + System.out.println("hive database:" + matcher3.group("database")); + String params = matcher9.group("params"); + if (null != params) { + String[] pairs = params.split("&"); + for (String pair : pairs) { + System.out.println("mysql params:" + pair); + } + } + } else { + System.out.println("error for hive!"); + } + + // 11、SQLite数据库 + // jdbc:sqlite:/tmp/phone.db + final Matcher matcher10 = JDBCURL.getPattern("jdbc:sqlite:{file}") + .matcher("jdbc:sqlite:D:\\Project\\Test\\phone.db"); + if (matcher10.matches()) { + System.out.println("sqlite file:" + matcher10.group("file")); } else { System.out.println("error for sqlite!"); } diff --git a/dbswitch-common/pom.xml b/dbswitch-common/pom.xml index 492f3cae..8b5cbb3a 100644 --- a/dbswitch-common/pom.xml +++ b/dbswitch-common/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.3 + 1.6.4 dbswitch-common diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/constant/Const.java b/dbswitch-common/src/main/java/com/gitee/dbswitch/common/constant/Const.java similarity index 97% rename from dbswitch-core/src/main/java/com/gitee/dbswitch/core/constant/Const.java rename to dbswitch-common/src/main/java/com/gitee/dbswitch/common/constant/Const.java index 4beb089f..f5b09143 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/constant/Const.java +++ b/dbswitch-common/src/main/java/com/gitee/dbswitch/common/constant/Const.java @@ -7,7 +7,7 @@ // Date : 2020/1/2 // Location: beijing , china ///////////////////////////////////////////////////////////// -package com.gitee.dbswitch.core.constant; +package com.gitee.dbswitch.common.constant; /** * 常量定义 diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/constant/DBTableType.java b/dbswitch-common/src/main/java/com/gitee/dbswitch/common/type/DBTableType.java similarity index 93% rename from dbswitch-core/src/main/java/com/gitee/dbswitch/core/constant/DBTableType.java rename to dbswitch-common/src/main/java/com/gitee/dbswitch/common/type/DBTableType.java index c4bae256..84f2fc16 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/constant/DBTableType.java +++ b/dbswitch-common/src/main/java/com/gitee/dbswitch/common/type/DBTableType.java @@ -7,7 +7,7 @@ // Date : 2020/1/2 // Location: beijing , china ///////////////////////////////////////////////////////////// -package com.gitee.dbswitch.core.constant; +package com.gitee.dbswitch.common.type; /** * 数据库表类型:视图表、物理表 diff --git a/dbswitch-common/src/main/java/com/gitee/dbswitch/common/type/DatabaseTypeEnum.java b/dbswitch-common/src/main/java/com/gitee/dbswitch/common/type/DatabaseTypeEnum.java index 8c77ebe2..c92a5e37 100644 --- a/dbswitch-common/src/main/java/com/gitee/dbswitch/common/type/DatabaseTypeEnum.java +++ b/dbswitch-common/src/main/java/com/gitee/dbswitch/common/type/DatabaseTypeEnum.java @@ -68,7 +68,13 @@ public enum DatabaseTypeEnum { /** * Kingbase数据库类型 */ - KINGBASE(10); + KINGBASE(10), + + /** + * HIVE数据库 + */ + HIVE(11), + ; private int index; diff --git a/dbswitch-common/src/main/java/com/gitee/dbswitch/common/util/CommonUtils.java b/dbswitch-common/src/main/java/com/gitee/dbswitch/common/util/CommonUtils.java index 09e824a4..5370db75 100644 --- a/dbswitch-common/src/main/java/com/gitee/dbswitch/common/util/CommonUtils.java +++ b/dbswitch-common/src/main/java/com/gitee/dbswitch/common/util/CommonUtils.java @@ -9,8 +9,8 @@ ///////////////////////////////////////////////////////////// package com.gitee.dbswitch.common.util; -import java.util.List; import com.gitee.dbswitch.common.type.DatabaseTypeEnum; +import java.util.List; /** * 普通工具类 @@ -32,7 +32,8 @@ public final class CommonUtils { */ public static String getTableFullNameByDatabase(DatabaseTypeEnum dbType, String schema, String table) { - if (dbType == DatabaseTypeEnum.MYSQL || dbType == DatabaseTypeEnum.MARIADB) { + if (dbType == DatabaseTypeEnum.MYSQL || dbType == DatabaseTypeEnum.MARIADB + || dbType == DatabaseTypeEnum.HIVE) { return String.format("`%s`.`%s`", schema, table); } else if (dbType == DatabaseTypeEnum.SQLSERVER || dbType == DatabaseTypeEnum.SQLSERVER2000) { return String.format("[%s].[%s]", schema, table); @@ -74,7 +75,8 @@ public final class CommonUtils { } private static String quoteString(DatabaseTypeEnum dbType, String keyName) { - if (dbType == DatabaseTypeEnum.MYSQL || dbType == DatabaseTypeEnum.MARIADB) { + if (dbType == DatabaseTypeEnum.MYSQL || dbType == DatabaseTypeEnum.MARIADB + || dbType == DatabaseTypeEnum.HIVE) { return String.format("`%s`", keyName); } else if (dbType == DatabaseTypeEnum.SQLSERVER || dbType == DatabaseTypeEnum.SQLSERVER2000) { return String.format("[%s]", keyName); diff --git a/dbswitch-common/src/main/java/com/gitee/dbswitch/common/util/HivePrepareUtils.java b/dbswitch-common/src/main/java/com/gitee/dbswitch/common/util/HivePrepareUtils.java new file mode 100644 index 00000000..af92816d --- /dev/null +++ b/dbswitch-common/src/main/java/com/gitee/dbswitch/common/util/HivePrepareUtils.java @@ -0,0 +1,52 @@ +package com.gitee.dbswitch.common.util; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +public final class HivePrepareUtils { + + private final static String HIVE_SQL_1 = "set hive.resultset.use.unique.column.names=false"; + private final static String HIVE_SQL_2 = "set hive.support.concurrency=true"; + private final static String HIVE_SQL_3 = "set hive.txn.manager = org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"; + + private HivePrepareUtils() { + } + + public static void prepare(Connection connection, String schema, String table) + throws SQLException { + executeWithoutResultSet(connection, HIVE_SQL_1); + if (isTransactionalTable(connection, schema, table)) { + executeWithoutResultSet(connection, HIVE_SQL_2); + executeWithoutResultSet(connection, HIVE_SQL_3); + } + } + + private static boolean isTransactionalTable(Connection connection, String schema, String table) + throws SQLException { + String fullTableName = String.format("`%s`.`%s`", schema, table); + String sql = String.format("DESCRIBE FORMATTED %s", fullTableName); + try (Statement st = connection.createStatement(); + ResultSet rs = st.executeQuery(sql)) { + while (rs.next()) { + String dataType = rs.getString("data_type"); + String comment = rs.getString("comment"); + if (dataType != null + && comment != null + && dataType.startsWith("transactional") + && comment.startsWith("true")) { + return true; + } + } + return false; + } + } + + private static boolean executeWithoutResultSet(Connection connection, String sql) + throws SQLException { + try (Statement st = connection.createStatement()) { + return st.execute(sql); + } + } +} diff --git a/dbswitch-core/pom.xml b/dbswitch-core/pom.xml index 90b9dbd0..cacaa02e 100644 --- a/dbswitch-core/pom.xml +++ b/dbswitch-core/pom.xml @@ -1,123 +1,136 @@ - 4.0.0 - - com.gitee.dbswitch - dbswitch-parent - 1.6.3 - + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 + + com.gitee.dbswitch + dbswitch-parent + 1.6.4 + - dbswitch-core + dbswitch-core - + - - com.gitee.dbswitch - dbswitch-common - ${project.version} - + + com.gitee.dbswitch + dbswitch-common + ${project.version} + - - mysql - mysql-connector-java - 5.1.47 - runtime - + + mysql + mysql-connector-java + 5.1.47 + runtime + - - org.postgresql - postgresql - runtime - + + org.postgresql + postgresql + runtime + - - com.oracle.ojdbc - ojdbc8 - runtime - + + com.oracle.ojdbc + ojdbc8 + runtime + - - - com.oracle.database.nls - orai18n - 19.7.0.0 - runtime - + + + com.oracle.database.nls + orai18n + 19.7.0.0 + runtime + - - com.microsoft.sqlserver - sqljdbc4 - 4.0 - runtime - + + com.microsoft.sqlserver + sqljdbc4 + 4.0 + runtime + - - com.microsoft.sqlserver - msbase - 3.0 - system - ${project.basedir}/lib/msbase.jar - + + com.microsoft.sqlserver + msbase + 3.0 + system + ${project.basedir}/lib/msbase.jar + - - com.microsoft.sqlserver - msutil - 3.0 - system - ${project.basedir}/lib/msutil.jar - + + com.microsoft.sqlserver + msutil + 3.0 + system + ${project.basedir}/lib/msutil.jar + - - com.microsoft.sqlserver - mssqlserver - 3.0 - system - ${project.basedir}/lib/mssqlserver.jar - + + com.microsoft.sqlserver + mssqlserver + 3.0 + system + ${project.basedir}/lib/mssqlserver.jar + - - com.pivotal - greenplum-jdbc - 5.1.4 - system - ${project.basedir}/lib/greenplum-jdbc-5.1.4.jar - + + com.pivotal + greenplum-jdbc + 5.1.4 + system + ${project.basedir}/lib/greenplum-jdbc-5.1.4.jar + - - com.dameng - dm-jdbc - 1.0.0 - system - ${project.basedir}/lib/DmJdbcDriver18.jar - + + com.dameng + dm-jdbc + 1.0.0 + system + ${project.basedir}/lib/DmJdbcDriver18.jar + - - com.kingbase - kingbase-jdbc - 8.2.0 - system - ${project.basedir}/lib/kingbase8-8.2.0.jar - + + com.kingbase + kingbase-jdbc + 8.2.0 + system + ${project.basedir}/lib/kingbase8-8.2.0.jar + - - org.mariadb.jdbc - mariadb-java-client - runtime - + + org.mariadb.jdbc + mariadb-java-client + runtime + - - com.ibm.db2.jcc - db2jcc - db2jcc4 - runtime - + + com.ibm.db2.jcc + db2jcc + db2jcc4 + runtime + - - org.apache.commons - commons-lang3 - + + org.apache.hive + hive-jdbc + 3.1.2 + runtime + + + jetty-runner + org.eclipse.jetty + + + - + + org.apache.commons + commons-lang3 + + + \ No newline at end of file diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/DatabaseFactory.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/DatabaseFactory.java index 356dfdb4..0a342288 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/DatabaseFactory.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/DatabaseFactory.java @@ -13,6 +13,7 @@ import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.core.database.impl.DatabaseDB2Impl; import com.gitee.dbswitch.core.database.impl.DatabaseDmImpl; import com.gitee.dbswitch.core.database.impl.DatabaseGreenplumImpl; +import com.gitee.dbswitch.core.database.impl.DatabaseHiveImpl; import com.gitee.dbswitch.core.database.impl.DatabaseKingbaseImpl; import com.gitee.dbswitch.core.database.impl.DatabaseMariaDBImpl; import com.gitee.dbswitch.core.database.impl.DatabaseMysqlImpl; @@ -45,6 +46,7 @@ public final class DatabaseFactory { put(DatabaseTypeEnum.DB2, DatabaseDB2Impl.class.getName()); put(DatabaseTypeEnum.DM, DatabaseDmImpl.class.getName()); put(DatabaseTypeEnum.KINGBASE, DatabaseKingbaseImpl.class.getName()); + put(DatabaseTypeEnum.HIVE, DatabaseHiveImpl.class.getName()); } }; @@ -64,4 +66,5 @@ public final class DatabaseFactory { private DatabaseFactory() { } + } diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDB2Impl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDB2Impl.java index 69ab7162..13ac3bf9 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDB2Impl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDB2Impl.java @@ -10,13 +10,12 @@ package com.gitee.dbswitch.core.database.impl; import com.gitee.dbswitch.common.type.DatabaseTypeEnum; -import com.gitee.dbswitch.core.constant.Const; +import com.gitee.dbswitch.common.constant.Const; import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnMetaData; import java.util.List; -import org.springframework.util.CollectionUtils; /** * 支持DB2数据库的元信息实现 @@ -113,7 +112,7 @@ public class DatabaseDB2Impl extends AbstractDatabase implements IDatabaseInterf retval += ")"; } - if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { + if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) { retval += " NOT NULL"; } diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDmImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDmImpl.java index bf8dfe47..62019e70 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDmImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDmImpl.java @@ -11,7 +11,7 @@ package com.gitee.dbswitch.core.database.impl; import java.util.List; import com.gitee.dbswitch.common.type.DatabaseTypeEnum; -import com.gitee.dbswitch.core.constant.Const; +import com.gitee.dbswitch.common.constant.Const; import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.model.ColumnDescription; diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseGreenplumImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseGreenplumImpl.java index fa1990a1..5299c603 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseGreenplumImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseGreenplumImpl.java @@ -10,13 +10,12 @@ package com.gitee.dbswitch.core.database.impl; import com.gitee.dbswitch.common.type.DatabaseTypeEnum; -import com.gitee.dbswitch.core.constant.Const; +import com.gitee.dbswitch.common.constant.Const; import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnMetaData; import java.util.List; -import org.springframework.util.CollectionUtils; /** * 支持Greenplum数据库的元信息实现 @@ -71,7 +70,7 @@ public class DatabaseGreenplumImpl extends AbstractDatabase implements IDatabase case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_INTEGER: case ColumnMetaData.TYPE_BIGNUMBER: - if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { + if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) { if (useAutoInc) { retval += "BIGSERIAL"; } else { diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseHiveImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseHiveImpl.java new file mode 100644 index 00000000..22e31ffc --- /dev/null +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseHiveImpl.java @@ -0,0 +1,154 @@ +package com.gitee.dbswitch.core.database.impl; + +import com.gitee.dbswitch.common.type.DatabaseTypeEnum; +import com.gitee.dbswitch.common.util.HivePrepareUtils; +import com.gitee.dbswitch.common.constant.Const; +import com.gitee.dbswitch.core.database.AbstractDatabase; +import com.gitee.dbswitch.core.database.IDatabaseInterface; +import com.gitee.dbswitch.core.model.ColumnDescription; +import com.gitee.dbswitch.core.model.ColumnMetaData; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class DatabaseHiveImpl extends AbstractDatabase implements IDatabaseInterface { + + public DatabaseHiveImpl() { + super("org.apache.hive.jdbc.HiveDriver"); + } + + @Override + public List queryTableColumnMeta(String schemaName, String tableName) { + String querySQL = this.getTableFieldsQuerySQL(schemaName, tableName); + List ret = new ArrayList<>(); + try (Statement st = connection.createStatement()) { + HivePrepareUtils.prepare(connection, schemaName, tableName); + try (ResultSet rs = st.executeQuery(querySQL)) { + ResultSetMetaData m = rs.getMetaData(); + int columns = m.getColumnCount(); + for (int i = 1; i <= columns; i++) { + String name = m.getColumnLabel(i); + if (null == name) { + name = m.getColumnName(i); + } + + ColumnDescription cd = new ColumnDescription(); + cd.setFieldName(name); + cd.setLabelName(name); + cd.setFieldType(m.getColumnType(i)); + if (0 != cd.getFieldType()) { + cd.setFieldTypeName(m.getColumnTypeName(i)); + cd.setFiledTypeClassName(m.getColumnClassName(i)); + cd.setDisplaySize(m.getColumnDisplaySize(i)); + cd.setPrecisionSize(m.getPrecision(i)); + cd.setScaleSize(m.getScale(i)); + cd.setAutoIncrement(m.isAutoIncrement(i)); + cd.setNullable(m.isNullable(i) != ResultSetMetaData.columnNoNulls); + } else { + // 处理视图中NULL as fieldName的情况 + cd.setFieldTypeName("CHAR"); + cd.setFiledTypeClassName(String.class.getName()); + cd.setDisplaySize(1); + cd.setPrecisionSize(1); + cd.setScaleSize(0); + cd.setAutoIncrement(false); + cd.setNullable(true); + } + + boolean signed = false; + try { + signed = m.isSigned(i); + } catch (Exception ignored) { + // This JDBC Driver doesn't support the isSigned method + // nothing more we can do here by catch the exception. + } + cd.setSigned(signed); + cd.setDbType(DatabaseTypeEnum.HIVE); + + ret.add(cd); + } + + return ret; + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public List queryTablePrimaryKeys(String schemaName, String tableName) { + return Collections.emptyList(); + } + + @Override + public List querySelectSqlColumnMeta(String sql) { + String querySQL = String.format(" %s LIMIT 1", sql.replace(";", "")); + return this.getSelectSqlColumnMeta(querySQL, DatabaseTypeEnum.HIVE); + } + + @Override + protected String getTableFieldsQuerySQL(String schemaName, String tableName) { + return String.format("SELECT * FROM `%s`.`%s` ", schemaName, tableName); + } + + @Override + protected String getTestQuerySQL(String sql) { + return String.format("explain %s", sql.replace(";", "")); + } + + @Override + public String getQuotedSchemaTableCombination(String schemaName, String tableName) { + return String.format(" `%s`.`%s` ", schemaName, tableName); + } + + @Override + public String getFieldDefinition(ColumnMetaData v, List pks, boolean useAutoInc, + boolean addCr) { + String fieldname = v.getName(); + int type = v.getType(); + + String retval = " `" + fieldname + "` "; + + switch (type) { + case ColumnMetaData.TYPE_TIMESTAMP: + case ColumnMetaData.TYPE_TIME: + retval += "TIMESTAMP"; + break; + case ColumnMetaData.TYPE_DATE: + retval += "DATE"; + break; + case ColumnMetaData.TYPE_BOOLEAN: + retval += "TINYINT"; + break; + case ColumnMetaData.TYPE_NUMBER: + retval += "FLOAT"; + break; + case ColumnMetaData.TYPE_INTEGER: + retval += "DECIMAL"; + break; + case ColumnMetaData.TYPE_BIGNUMBER: + retval += "BIGINT"; + break; + case ColumnMetaData.TYPE_STRING: + retval += "STRING"; + break; + case ColumnMetaData.TYPE_BINARY: + retval += "BINARY"; + break; + default: + retval += "STRING"; + break; + } + + if (addCr) { + retval += Const.CR; + } + + return retval; + } + +} diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseKingbaseImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseKingbaseImpl.java index 3b910a4a..33034e02 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseKingbaseImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseKingbaseImpl.java @@ -10,13 +10,12 @@ package com.gitee.dbswitch.core.database.impl; import com.gitee.dbswitch.common.type.DatabaseTypeEnum; -import com.gitee.dbswitch.core.constant.Const; +import com.gitee.dbswitch.common.constant.Const; import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnMetaData; import java.util.List; -import org.springframework.util.CollectionUtils; /** * 支持Kingbase数据库的元信息实现 @@ -71,7 +70,7 @@ public class DatabaseKingbaseImpl extends AbstractDatabase implements IDatabaseI case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_INTEGER: case ColumnMetaData.TYPE_BIGNUMBER: - if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { + if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) { if (useAutoInc) { retval += "BIGSERIAL"; } else { diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseMysqlImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseMysqlImpl.java index 39dfa092..e2a5d545 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseMysqlImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseMysqlImpl.java @@ -10,7 +10,7 @@ package com.gitee.dbswitch.core.database.impl; import com.gitee.dbswitch.common.type.DatabaseTypeEnum; -import com.gitee.dbswitch.core.constant.Const; +import com.gitee.dbswitch.common.constant.Const; import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.model.ColumnDescription; @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; /** * 支持MySQL数据库的元信息实现 @@ -131,7 +130,7 @@ public class DatabaseMysqlImpl extends AbstractDatabase implements IDatabaseInte case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_INTEGER: case ColumnMetaData.TYPE_BIGNUMBER: - if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { + if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) { if (useAutoInc) { retval += "BIGINT AUTO_INCREMENT NOT NULL"; } else { @@ -174,7 +173,7 @@ public class DatabaseMysqlImpl extends AbstractDatabase implements IDatabaseInte retval += "CHAR(1)"; } else if (length < 256) { retval += "VARCHAR(" + length + ")"; - } else if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { + } else if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) { /* * MySQL5.6中varchar字段为主键时最大长度为254,例如如下的建表语句在MySQL5.7下能通过，但在MySQL5.6下无法通过： * create table `t_test`( @@ -212,7 +211,7 @@ public class DatabaseMysqlImpl extends AbstractDatabase implements IDatabaseInte @Override public String getPrimaryKeyAsString(List pks) { - if (!CollectionUtils.isEmpty(pks)) { + if (null != pks && !pks.isEmpty()) { StringBuilder sb = new StringBuilder(); sb.append("`"); sb.append(StringUtils.join(pks, "` , `")); diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseOracleImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseOracleImpl.java index 7d81b9a6..2b96c507 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseOracleImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseOracleImpl.java @@ -10,7 +10,7 @@ package com.gitee.dbswitch.core.database.impl; import com.gitee.dbswitch.common.type.DatabaseTypeEnum; -import com.gitee.dbswitch.core.constant.Const; +import com.gitee.dbswitch.common.constant.Const; import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.model.ColumnDescription; diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabasePostgresImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabasePostgresImpl.java index 6d91f451..4a89d379 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabasePostgresImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabasePostgresImpl.java @@ -10,13 +10,12 @@ package com.gitee.dbswitch.core.database.impl; import com.gitee.dbswitch.common.type.DatabaseTypeEnum; -import com.gitee.dbswitch.core.constant.Const; +import com.gitee.dbswitch.common.constant.Const; import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnMetaData; import java.util.List; -import org.springframework.util.CollectionUtils; /** * 支持PostgreSQL数据库的元信息实现 @@ -71,7 +70,7 @@ public class DatabasePostgresImpl extends AbstractDatabase implements IDatabaseI case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_INTEGER: case ColumnMetaData.TYPE_BIGNUMBER: - if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { + if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) { if (useAutoInc) { retval += "BIGSERIAL"; } else { @@ -107,7 +106,7 @@ public class DatabasePostgresImpl extends AbstractDatabase implements IDatabaseI if (length < 1 || length >= AbstractDatabase.CLOB_LENGTH) { retval += "TEXT"; } else { - if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { + if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) { retval += "VARCHAR(" + length + ")"; } else { retval += "TEXT"; diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseSqlserverImpl.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseSqlserverImpl.java index 06393ac4..05d9ace1 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseSqlserverImpl.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseSqlserverImpl.java @@ -10,7 +10,7 @@ package com.gitee.dbswitch.core.database.impl; import com.gitee.dbswitch.common.type.DatabaseTypeEnum; -import com.gitee.dbswitch.core.constant.Const; +import com.gitee.dbswitch.common.constant.Const; import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.model.ColumnDescription; @@ -24,7 +24,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import org.apache.commons.lang3.StringUtils; -import org.springframework.util.CollectionUtils; /** * 支持SQLServer数据库的元信息实现 @@ -195,7 +194,7 @@ public class DatabaseSqlserverImpl extends AbstractDatabase implements IDatabase case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_INTEGER: case ColumnMetaData.TYPE_BIGNUMBER: - if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { + if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) { if (useAutoInc) { retval += "BIGINT IDENTITY(0,1)"; } else { @@ -255,7 +254,7 @@ public class DatabaseSqlserverImpl extends AbstractDatabase implements IDatabase @Override public String getPrimaryKeyAsString(List pks) { - if (!CollectionUtils.isEmpty(pks)) { + if (null != pks && !pks.isEmpty()) { StringBuilder sb = new StringBuilder(); sb.append("["); sb.append(StringUtils.join(pks, "] , [")); diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/model/TableDescription.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/model/TableDescription.java index 36c19d03..d3dcaa5d 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/model/TableDescription.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/model/TableDescription.java @@ -9,7 +9,7 @@ ///////////////////////////////////////////////////////////// package com.gitee.dbswitch.core.model; -import com.gitee.dbswitch.core.constant.DBTableType; +import com.gitee.dbswitch.common.type.DBTableType; /** * 数据库表描述符信息定义(Table Description) diff --git a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/util/GenerateSqlUtils.java b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/util/GenerateSqlUtils.java index 928b8e38..8ca40aeb 100644 --- a/dbswitch-core/src/main/java/com/gitee/dbswitch/core/util/GenerateSqlUtils.java +++ b/dbswitch-core/src/main/java/com/gitee/dbswitch/core/util/GenerateSqlUtils.java @@ -10,7 +10,7 @@ package com.gitee.dbswitch.core.util; import com.gitee.dbswitch.common.type.DatabaseTypeEnum; -import com.gitee.dbswitch.core.constant.Const; +import com.gitee.dbswitch.common.constant.Const; import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.DatabaseFactory; import com.gitee.dbswitch.core.model.ColumnDescription; diff --git a/dbswitch-data/pom.xml b/dbswitch-data/pom.xml index 25d48bac..ff1f0907 100644 --- a/dbswitch-data/pom.xml +++ b/dbswitch-data/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.3 + 1.6.4 dbswitch-data @@ -57,6 +57,12 @@ org.springframework.boot spring-boot-starter-jdbc + + + spring-boot-starter-logging + org.springframework.boot + + diff --git a/dbswitch-data/src/main/java/com/gitee/dbswitch/data/handler/MigrationHandler.java b/dbswitch-data/src/main/java/com/gitee/dbswitch/data/handler/MigrationHandler.java index 0947247c..f7b6cd0b 100644 --- a/dbswitch-data/src/main/java/com/gitee/dbswitch/data/handler/MigrationHandler.java +++ b/dbswitch-data/src/main/java/com/gitee/dbswitch/data/handler/MigrationHandler.java @@ -204,9 +204,9 @@ public class MigrationHandler implements Supplier { .getDatabaseProduceName(sourceDataSource); String fullTableName = CommonUtils.getTableFullNameByDatabase(sourceDatabaseType, tableDescription.getSchemaName(), tableDescription.getTableName()); - Map columnMetaData = JdbcTemplateUtils - .getColumnMetaData(new JdbcTemplate(sourceDataSource), - fullTableName); + Map columnMetaData = JdbcTemplateUtils.getColumnMetaData( + sourceDataSource, sourceDatabaseType, tableDescription.getSchemaName(), + tableDescription.getTableName()); List fields = new ArrayList<>(columnMetaData.keySet()); StatementResultSet srs = sourceOperator diff --git a/dbswitch-data/src/main/java/com/gitee/dbswitch/data/util/DataSourceUtils.java b/dbswitch-data/src/main/java/com/gitee/dbswitch/data/util/DataSourceUtils.java index 88c52cda..3380a6f4 100644 --- a/dbswitch-data/src/main/java/com/gitee/dbswitch/data/util/DataSourceUtils.java +++ b/dbswitch-data/src/main/java/com/gitee/dbswitch/data/util/DataSourceUtils.java @@ -53,6 +53,10 @@ public final class DataSourceUtils { */ public static HikariDataSource createTargetDataSource( DbswichProperties.TargetDataSourceProperties description) { + if (description.getUrl().trim().startsWith("jdbc:hive2://")) { + throw new UnsupportedOperationException("Unsupported hive as target datasource!!!"); + } + HikariDataSource ds = new HikariDataSource(); ds.setPoolName("The_Target_DB_Connection"); ds.setJdbcUrl(description.getUrl()); diff --git a/dbswitch-data/src/main/java/com/gitee/dbswitch/data/util/JdbcTemplateUtils.java b/dbswitch-data/src/main/java/com/gitee/dbswitch/data/util/JdbcTemplateUtils.java index b06c08e3..a918559c 100644 --- a/dbswitch-data/src/main/java/com/gitee/dbswitch/data/util/JdbcTemplateUtils.java +++ b/dbswitch-data/src/main/java/com/gitee/dbswitch/data/util/JdbcTemplateUtils.java @@ -10,10 +10,13 @@ package com.gitee.dbswitch.data.util; import com.gitee.dbswitch.common.type.DatabaseTypeEnum; +import com.gitee.dbswitch.common.util.CommonUtils; import com.gitee.dbswitch.dbcommon.util.DatabaseAwareUtils; +import com.gitee.dbswitch.common.util.HivePrepareUtils; import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; +import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; import java.util.Map; @@ -48,6 +51,8 @@ public final class JdbcTemplateUtils { return DatabaseTypeEnum.DM; } else if (productName.equalsIgnoreCase("Kingbase")) { return DatabaseTypeEnum.KINGBASE; + } else if (productName.equalsIgnoreCase("Hive")) { + return DatabaseTypeEnum.HIVE; } else { DatabaseDriver databaseDriver = DatabaseDriver.fromProductName(productName); if (DatabaseDriver.MARIADB == databaseDriver) { @@ -71,30 +76,36 @@ public final class JdbcTemplateUtils { /** * 获取表字段的元信息 * - * @param sourceJdbcTemplate JdbcTemplate - * @param fullTableName 表的全名 + * @param dataSource DataSource + * @param databaseType databaseType + * @param schemaName schemaName + * @param tableName tableName * @return Map */ public static Map getColumnMetaData( - JdbcTemplate sourceJdbcTemplate, - String fullTableName) { + DataSource dataSource, DatabaseTypeEnum databaseType, + String schemaName, String tableName) { + String fullTableName = CommonUtils.getTableFullNameByDatabase(databaseType, + schemaName, tableName); final String sql = String.format("select * from %s where 1=2", fullTableName); Map columnMetaData = new HashMap<>(); - sourceJdbcTemplate.execute((Connection connection) -> { - try (Statement stmt = connection.createStatement(); - ResultSet rs = stmt.executeQuery(sql)) { - ResultSetMetaData rsMetaData = rs.getMetaData(); - for (int i = 0, len = rsMetaData.getColumnCount(); i < len; i++) { - columnMetaData.put(rsMetaData.getColumnName(i + 1), rsMetaData.getColumnType(i + 1)); + try (Connection connection = dataSource.getConnection()) { + try (Statement stmt = connection.createStatement()) { + if (connection.getMetaData().getDatabaseProductName().contains("Hive")) { + HivePrepareUtils.prepare(connection, schemaName, tableName); + } + try (ResultSet rs = stmt.executeQuery(sql)) { + ResultSetMetaData rsMetaData = rs.getMetaData(); + for (int i = 0, len = rsMetaData.getColumnCount(); i < len; i++) { + columnMetaData.put(rsMetaData.getColumnName(i + 1), rsMetaData.getColumnType(i + 1)); + } + return columnMetaData; } - return true; - } catch (Exception e) { - throw new RuntimeException( - String.format("获取表:%s 的字段的元信息时失败. 请联系 DBA 核查该库、表信息.", fullTableName), e); } - }); - - return columnMetaData; + } catch (SQLException e) { + throw new RuntimeException( + String.format("获取表:%s 的字段的元信息时失败. 请联系 DBA 核查该库、表信息.", fullTableName), e); + } } /** @@ -112,7 +123,8 @@ public final class JdbcTemplateUtils { String sql = "SELECT count(*) as total FROM information_schema.tables " + "WHERE table_schema=? AND table_name=? AND ENGINE='InnoDB'"; JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); - return jdbcTemplate.queryForObject(sql, new Object[]{schemaName, tableName}, Integer.class) > 0; + return jdbcTemplate.queryForObject(sql, new Object[]{schemaName, tableName}, Integer.class) + > 0; } } diff --git a/dbswitch-dbchange/pom.xml b/dbswitch-dbchange/pom.xml index aff23d4a..a519ee64 100644 --- a/dbswitch-dbchange/pom.xml +++ b/dbswitch-dbchange/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.3 + 1.6.4 dbswitch-dbchange diff --git a/dbswitch-dbcommon/pom.xml b/dbswitch-dbcommon/pom.xml index 87edd080..37358d90 100644 --- a/dbswitch-dbcommon/pom.xml +++ b/dbswitch-dbcommon/pom.xml @@ -1,27 +1,40 @@ - + 4.0.0 com.gitee.dbswitch dbswitch-parent - 1.6.3 + 1.6.4 - + dbswitch-dbcommon - - - - org.springframework.boot - spring-boot-starter-jdbc - - - org.apache.commons - commons-lang3 - - - org.projectlombok - lombok - provided - - - + + + + com.gitee.dbswitch + dbswitch-common + ${project.version} + + + org.springframework.boot + spring-boot-starter-jdbc + + + spring-boot-starter-logging + org.springframework.boot + + + + + org.apache.commons + commons-lang3 + + + org.projectlombok + lombok + provided + + + \ No newline at end of file diff --git a/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/AbstractDatabaseOperator.java b/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/AbstractDatabaseOperator.java index 06de9cf5..4c180c2e 100644 --- a/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/AbstractDatabaseOperator.java +++ b/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/AbstractDatabaseOperator.java @@ -12,9 +12,9 @@ package com.gitee.dbswitch.dbcommon.database; import com.gitee.dbswitch.dbcommon.constant.Constants; import com.gitee.dbswitch.dbcommon.domain.StatementResultSet; import java.sql.Connection; -import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.Objects; import javax.sql.DataSource; import lombok.extern.slf4j.Slf4j; @@ -94,8 +94,8 @@ public abstract class AbstractDatabaseOperator implements IDatabaseOperator { log.debug("Execute sql :{}", sql); } try (Connection connection = dataSource.getConnection(); - PreparedStatement ps = connection.prepareStatement(sql)) { - return ps.executeUpdate(); + Statement st = connection.createStatement()) { + return st.executeUpdate(sql); } catch (SQLException e) { throw new RuntimeException(e); } diff --git a/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/DatabaseOperatorFactory.java b/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/DatabaseOperatorFactory.java index ca22fe84..6ec39cee 100644 --- a/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/DatabaseOperatorFactory.java +++ b/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/DatabaseOperatorFactory.java @@ -12,6 +12,7 @@ package com.gitee.dbswitch.dbcommon.database; import com.gitee.dbswitch.dbcommon.database.impl.DB2DatabaseOperator; import com.gitee.dbswitch.dbcommon.database.impl.DmDatabaseOperator; import com.gitee.dbswitch.dbcommon.database.impl.GreenplumDatabaseOperator; +import com.gitee.dbswitch.dbcommon.database.impl.HiveDatabaseOperator; import com.gitee.dbswitch.dbcommon.database.impl.KingbaseDatabaseOperator; import com.gitee.dbswitch.dbcommon.database.impl.MysqlDatabaseOperator; import com.gitee.dbswitch.dbcommon.database.impl.OracleDatabaseOperator; @@ -43,6 +44,7 @@ public final class DatabaseOperatorFactory { put("DB2", DB2DatabaseOperator::new); put("DM", DmDatabaseOperator::new); put("KINGBASE", KingbaseDatabaseOperator::new); + put("HIVE", HiveDatabaseOperator::new); } }; @@ -56,7 +58,7 @@ public final class DatabaseOperatorFactory { String type = DatabaseAwareUtils.getDatabaseNameByDataSource(dataSource).toUpperCase(); if (!DATABASE_OPERATOR_MAPPER.containsKey(type)) { throw new RuntimeException( - String.format("[dbcommon] Unknown Supported database type (%s)", type)); + String.format("[dbcommon] Unsupported database type (%s)", type)); } return DATABASE_OPERATOR_MAPPER.get(type).apply(dataSource); diff --git a/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/impl/HiveDatabaseOperator.java b/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/impl/HiveDatabaseOperator.java new file mode 100644 index 00000000..cd63c568 --- /dev/null +++ b/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/impl/HiveDatabaseOperator.java @@ -0,0 +1,61 @@ +package com.gitee.dbswitch.dbcommon.database.impl; + +import com.gitee.dbswitch.common.util.HivePrepareUtils; +import com.gitee.dbswitch.dbcommon.constant.Constants; +import com.gitee.dbswitch.dbcommon.domain.StatementResultSet; +import java.sql.Connection; +import java.sql.ResultSet; +import java.util.List; +import javax.sql.DataSource; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class HiveDatabaseOperator extends MysqlDatabaseOperator { + + public HiveDatabaseOperator(DataSource dataSource) { + super(dataSource); + } + + @Override + public StatementResultSet queryTableData(String schemaName, String tableName, List fields, + List orders) { + String sql = String.format("select `%s` from `%s`.`%s` order by `%s` asc ", + StringUtils.join(fields, "`,`"), + schemaName, tableName, StringUtils.join(orders, "`,`")); + return selectHiveTableData(sql, schemaName, tableName); + } + + @Override + public StatementResultSet queryTableData(String schemaName, String tableName, + List fields) { + String sql = String.format("select `%s` from `%s`.`%s` ", + StringUtils.join(fields, "`,`"), schemaName, tableName); + return selectHiveTableData(sql, schemaName, tableName); + } + + private StatementResultSet selectHiveTableData(String sql, String schemaName, String tableName) { + if (log.isDebugEnabled()) { + log.debug("Query table data sql :{}", sql); + } + + try { + Connection connection = dataSource.getConnection(); + HivePrepareUtils.prepare(connection, schemaName, tableName); + + StatementResultSet srs = new StatementResultSet(); + srs.setConnection(connection); + srs.setAutoCommit(srs.getConnection().getAutoCommit()); + srs.getConnection().setAutoCommit(false); + srs.setStatement(srs.getConnection() + .createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)); + srs.getStatement().setQueryTimeout(Constants.DEFAULT_QUERY_TIMEOUT_SECONDS); + srs.getStatement().setFetchSize(this.fetchSize); + srs.setResultset(srs.getStatement().executeQuery(sql)); + return srs; + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + +} diff --git a/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/util/DatabaseAwareUtils.java b/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/util/DatabaseAwareUtils.java index 01b95933..a5be7930 100644 --- a/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/util/DatabaseAwareUtils.java +++ b/dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/util/DatabaseAwareUtils.java @@ -45,6 +45,8 @@ public final class DatabaseAwareUtils { return "kingbase"; } else if (productName.equalsIgnoreCase("DM DBMS")) { return "dm"; + } else if (productName.equalsIgnoreCase("Apache Hive")) { + return "hive"; } DatabaseDriver databaseDriver = DatabaseDriver.fromProductName(productName); diff --git a/dbswitch-dbsynch/pom.xml b/dbswitch-dbsynch/pom.xml index e8b7c35e..7f9b5f3e 100644 --- a/dbswitch-dbsynch/pom.xml +++ b/dbswitch-dbsynch/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.3 + 1.6.4 dbswitch-dbsynch diff --git a/dbswitch-dbsynch/src/main/java/com/gitee/dbswitch/dbsynch/DatabaseSynchronizeFactory.java b/dbswitch-dbsynch/src/main/java/com/gitee/dbswitch/dbsynch/DatabaseSynchronizeFactory.java index e9ec0c61..3daecbb0 100644 --- a/dbswitch-dbsynch/src/main/java/com/gitee/dbswitch/dbsynch/DatabaseSynchronizeFactory.java +++ b/dbswitch-dbsynch/src/main/java/com/gitee/dbswitch/dbsynch/DatabaseSynchronizeFactory.java @@ -56,7 +56,7 @@ public final class DatabaseSynchronizeFactory { String type = DatabaseAwareUtils.getDatabaseNameByDataSource(dataSource).toUpperCase(); if (!DATABASE_SYNC_MAPPER.containsKey(type)) { throw new RuntimeException( - String.format("[dbsynch] Unknown Supported database type (%s)", type)); + String.format("[dbsynch] Unsupported database type (%s)", type)); } return DATABASE_SYNC_MAPPER.get(type).apply(dataSource); diff --git a/dbswitch-dbwriter/pom.xml b/dbswitch-dbwriter/pom.xml index 2365bd3c..8dd56d03 100644 --- a/dbswitch-dbwriter/pom.xml +++ b/dbswitch-dbwriter/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.3 + 1.6.4 dbswitch-dbwriter @@ -14,6 +14,12 @@ org.springframework.boot spring-boot-starter-jdbc + + + spring-boot-starter-logging + org.springframework.boot + + org.apache.commons diff --git a/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/DatabaseWriterFactory.java b/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/DatabaseWriterFactory.java index 65de1afe..6fbf5ec2 100644 --- a/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/DatabaseWriterFactory.java +++ b/dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/DatabaseWriterFactory.java @@ -66,7 +66,7 @@ public class DatabaseWriterFactory { if (!DATABASE_WRITER_MAPPER.containsKey(type.trim())) { throw new RuntimeException( - String.format("[dbwrite] Unknown Supported database type (%s)", type)); + String.format("[dbwrite] Unsupported database type (%s)", type)); } return DATABASE_WRITER_MAPPER.get(type.trim()).apply(dataSource); diff --git a/dbswitch-pgwriter/pom.xml b/dbswitch-pgwriter/pom.xml index a6ecc42b..9be0f92d 100644 --- a/dbswitch-pgwriter/pom.xml +++ b/dbswitch-pgwriter/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.3 + 1.6.4 dbswitch-pgwriter diff --git a/dbswitch-sql/pom.xml b/dbswitch-sql/pom.xml index d2cceb22..739eae31 100644 --- a/dbswitch-sql/pom.xml +++ b/dbswitch-sql/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.3 + 1.6.4 dbswitch-sql diff --git a/package-tool/pom.xml b/package-tool/pom.xml index 9427ed74..0780d4bf 100644 --- a/package-tool/pom.xml +++ b/package-tool/pom.xml @@ -5,7 +5,7 @@ com.gitee.dbswitch dbswitch-parent - 1.6.3 + 1.6.4 package-tool diff --git a/pom.xml b/pom.xml index 1651a137..7b482f26 100644 --- a/pom.xml +++ b/pom.xml @@ -2,16 +2,9 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - - org.springframework.boot - spring-boot-starter-parent - 2.2.0.RELEASE - - - com.gitee.dbswitch dbswitch-parent - 1.6.3 + 1.6.4 pom dbswitch database switch project @@ -24,6 +17,7 @@ 1.8 UTF-8 UTF-8 + 2.3.7.RELEASE @@ -40,12 +34,17 @@ package-tool - - - org.springframework.boot - spring-boot-starter - - + + + + org.springframework.boot + spring-boot-dependencies + ${spring-boot.version} + pom + import + + + diff --git a/version.cmd b/version.cmd index b27a0d36..ce62f0cc 100644 --- a/version.cmd +++ b/version.cmd @@ -1,6 +1,6 @@ @echo off -set APP_VERSION=1.6.3 +set APP_VERSION=1.6.4 echo "Clean Project ..." call mvn clean -f pom.xml