dromara/dbswitch
1
0
Fork 0
mirror of https://gitee.com/dromara/dbswitch.git synced 2025-09-10 05:59:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

support hive

Browse Source
This commit is contained in:
inrgihc
2022-02-02 14:12:17 +08:00
parent c187bef98c
commit 8559e2ebe9
41 changed files with 567 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
dbswitch-admin/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
</parent> </parent>
<artifactId>dbswitch-admin</artifactId> <artifactId>dbswitch-admin</artifactId>
@@ -39,6 +39,12 @@
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId> <artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<artifactId>spring-boot-starter-logging</artifactId>
<groupId>org.springframework.boot</groupId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>

2
dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/type/SupportDbTypeEnum.java
View File

@@ -34,6 +34,8 @@ public enum SupportDbTypeEnum {
new String[]{"jdbc:dm://{host}:{port}[/{database}][\\?{params}]"}), new String[]{"jdbc:dm://{host}:{port}[/{database}][\\?{params}]"}),
KINGBASE(8, "kingbase", "com.kingbase8.Driver", 54321, "SELECT 1", KINGBASE(8, "kingbase", "com.kingbase8.Driver", 54321, "SELECT 1",
new String[]{"jdbc:kingbase8://{host}[:{port}]/[{database}][\\?{params}]"}), new String[]{"jdbc:kingbase8://{host}[:{port}]/[{database}][\\?{params}]"}),
HIVE(1, "hive", "org.apache.hive.jdbc.HiveDriver", 10000, "SELECT 1",
new String[]{"jdbc:hive2://{host}[:{port}]/[{database}][\\?{params}]"}),
; ;
private int id; private int id;

30
dbswitch-admin/src/main/java/com/gitee/dbswitch/admin/util/JDBCURL.java
View File

@@ -283,12 +283,32 @@ public class JDBCURL {
System.out.println("error for db2!"); System.out.println("error for db2!");
} }
// 9、SQLite数据库 // 10、Hive数据库
// jdbc:sqlite:/tmp/phone.db // jdbc:hive2://172.17.2.10:10000/test?useUnicode=true&useSSL=false
final Matcher matcher9 = JDBCURL.getPattern("jdbc:sqlite:{file}") final Matcher matcher9 = JDBCURL
.matcher("jdbc:sqlite:D:\\Project\\Test\\phone.db"); .getPattern("jdbc:hive2://{host}[:{port}]/[{database}][\\?{params}]")
.matcher("jdbc:hive2://127.0.0.1:10000/default?useUnicode=true&useSSL=false");
if (matcher9.matches()) { if (matcher9.matches()) {
System.out.println("sqlite file:" + matcher9.group("file")); System.out.println("hive host:" + matcher3.group("host"));
System.out.println("hive port:" + matcher3.group("port"));
System.out.println("hive database:" + matcher3.group("database"));
String params = matcher9.group("params");
if (null != params) {
String[] pairs = params.split("&");
for (String pair : pairs) {
System.out.println("mysql params:" + pair);
}
}
} else {
System.out.println("error for hive!");
}
// 11、SQLite数据库
// jdbc:sqlite:/tmp/phone.db
final Matcher matcher10 = JDBCURL.getPattern("jdbc:sqlite:{file}")
.matcher("jdbc:sqlite:D:\\Project\\Test\\phone.db");
if (matcher10.matches()) {
System.out.println("sqlite file:" + matcher10.group("file"));
} else { } else {
System.out.println("error for sqlite!"); System.out.println("error for sqlite!");
} }

2
dbswitch-common/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
</parent> </parent>
<artifactId>dbswitch-common</artifactId> <artifactId>dbswitch-common</artifactId>

2
dbswitch-core/src/main/java/com/gitee/dbswitch/core/constant/Const.java → dbswitch-common/src/main/java/com/gitee/dbswitch/common/constant/Const.java
View File

@@ -7,7 +7,7 @@
// Date : 2020/1/2 // Date : 2020/1/2
// Location: beijing , china // Location: beijing , china
///////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////
package com.gitee.dbswitch.core.constant; package com.gitee.dbswitch.common.constant;
/** /**
* 常量定义 * 常量定义

2
dbswitch-core/src/main/java/com/gitee/dbswitch/core/constant/DBTableType.java → dbswitch-common/src/main/java/com/gitee/dbswitch/common/type/DBTableType.java
View File

@@ -7,7 +7,7 @@
// Date : 2020/1/2 // Date : 2020/1/2
// Location: beijing , china // Location: beijing , china
///////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////
package com.gitee.dbswitch.core.constant; package com.gitee.dbswitch.common.type;
/** /**
* 数据库表类型:视图表、物理表 * 数据库表类型:视图表、物理表

8
dbswitch-common/src/main/java/com/gitee/dbswitch/common/type/DatabaseTypeEnum.java
View File

@@ -68,7 +68,13 @@ public enum DatabaseTypeEnum {
/** /**
* Kingbase数据库类型 * Kingbase数据库类型
*/ */
KINGBASE(10); KINGBASE(10),
/**
* HIVE数据库
*/
HIVE(11),
;
private int index; private int index;

8
dbswitch-common/src/main/java/com/gitee/dbswitch/common/util/CommonUtils.java
View File

@@ -9,8 +9,8 @@
///////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////
package com.gitee.dbswitch.common.util; package com.gitee.dbswitch.common.util;
import java.util.List;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import java.util.List;
/** /**
* 普通工具类 * 普通工具类
@@ -32,7 +32,8 @@ public final class CommonUtils {
*/ */
public static String getTableFullNameByDatabase(DatabaseTypeEnum dbType, String schema, public static String getTableFullNameByDatabase(DatabaseTypeEnum dbType, String schema,
String table) { String table) {
if (dbType == DatabaseTypeEnum.MYSQL || dbType == DatabaseTypeEnum.MARIADB) { if (dbType == DatabaseTypeEnum.MYSQL || dbType == DatabaseTypeEnum.MARIADB
|| dbType == DatabaseTypeEnum.HIVE) {
return String.format("`%s`.`%s`", schema, table); return String.format("`%s`.`%s`", schema, table);
} else if (dbType == DatabaseTypeEnum.SQLSERVER || dbType == DatabaseTypeEnum.SQLSERVER2000) { } else if (dbType == DatabaseTypeEnum.SQLSERVER || dbType == DatabaseTypeEnum.SQLSERVER2000) {
return String.format("[%s].[%s]", schema, table); return String.format("[%s].[%s]", schema, table);
@@ -74,7 +75,8 @@ public final class CommonUtils {
} }
private static String quoteString(DatabaseTypeEnum dbType, String keyName) { private static String quoteString(DatabaseTypeEnum dbType, String keyName) {
if (dbType == DatabaseTypeEnum.MYSQL || dbType == DatabaseTypeEnum.MARIADB) { if (dbType == DatabaseTypeEnum.MYSQL || dbType == DatabaseTypeEnum.MARIADB
|| dbType == DatabaseTypeEnum.HIVE) {
return String.format("`%s`", keyName); return String.format("`%s`", keyName);
} else if (dbType == DatabaseTypeEnum.SQLSERVER || dbType == DatabaseTypeEnum.SQLSERVER2000) { } else if (dbType == DatabaseTypeEnum.SQLSERVER || dbType == DatabaseTypeEnum.SQLSERVER2000) {
return String.format("[%s]", keyName); return String.format("[%s]", keyName);

52
dbswitch-common/src/main/java/com/gitee/dbswitch/common/util/HivePrepareUtils.java Normal file
View File

@@ -0,0 +1,52 @@
package com.gitee.dbswitch.common.util;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public final class HivePrepareUtils {
private final static String HIVE_SQL_1 = "set hive.resultset.use.unique.column.names=false";
private final static String HIVE_SQL_2 = "set hive.support.concurrency=true";
private final static String HIVE_SQL_3 = "set hive.txn.manager = org.apache.hadoop.hive.ql.lockmgr.DbTxnManager";
private HivePrepareUtils() {
}
public static void prepare(Connection connection, String schema, String table)
throws SQLException {
executeWithoutResultSet(connection, HIVE_SQL_1);
if (isTransactionalTable(connection, schema, table)) {
executeWithoutResultSet(connection, HIVE_SQL_2);
executeWithoutResultSet(connection, HIVE_SQL_3);
}
}
private static boolean isTransactionalTable(Connection connection, String schema, String table)
throws SQLException {
String fullTableName = String.format("`%s`.`%s`", schema, table);
String sql = String.format("DESCRIBE FORMATTED %s", fullTableName);
try (Statement st = connection.createStatement();
ResultSet rs = st.executeQuery(sql)) {
while (rs.next()) {
String dataType = rs.getString("data_type");
String comment = rs.getString("comment");
if (dataType != null
&& comment != null
&& dataType.startsWith("transactional")
&& comment.startsWith("true")) {
return true;
}
}
return false;
}
}
private static boolean executeWithoutResultSet(Connection connection, String sql)
throws SQLException {
try (Statement st = connection.createStatement()) {
return st.execute(sql);
}
}
}

217
dbswitch-core/pom.xml
View File

@@ -1,123 +1,136 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" <project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
</parent> </parent>
<artifactId>dbswitch-core</artifactId> <artifactId>dbswitch-core</artifactId>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-common</artifactId> <artifactId>dbswitch-common</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>mysql</groupId> <groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId> <artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version> <version>5.1.47</version>
<scope>runtime</scope> <scope>runtime</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.postgresql</groupId> <groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId> <artifactId>postgresql</artifactId>
<scope>runtime</scope> <scope>runtime</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.oracle.ojdbc</groupId> <groupId>com.oracle.ojdbc</groupId>
<artifactId>ojdbc8</artifactId> <artifactId>ojdbc8</artifactId>
<scope>runtime</scope> <scope>runtime</scope>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/com.oracle.database.nls/orai18n --> <!-- https://mvnrepository.com/artifact/com.oracle.database.nls/orai18n -->
<dependency> <dependency>
<groupId>com.oracle.database.nls</groupId> <groupId>com.oracle.database.nls</groupId>
<artifactId>orai18n</artifactId> <artifactId>orai18n</artifactId>
<version>19.7.0.0</version> <version>19.7.0.0</version>
<scope>runtime</scope> <scope>runtime</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.microsoft.sqlserver</groupId> <groupId>com.microsoft.sqlserver</groupId>
<artifactId>sqljdbc4</artifactId> <artifactId>sqljdbc4</artifactId>
<version>4.0</version> <version>4.0</version>
<scope>runtime</scope> <scope>runtime</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.microsoft.sqlserver</groupId> <groupId>com.microsoft.sqlserver</groupId>
<artifactId>msbase</artifactId> <artifactId>msbase</artifactId>
<version>3.0</version> <version>3.0</version>
<scope>system</scope> <scope>system</scope>
<systemPath>${project.basedir}/lib/msbase.jar</systemPath> <systemPath>${project.basedir}/lib/msbase.jar</systemPath>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.microsoft.sqlserver</groupId> <groupId>com.microsoft.sqlserver</groupId>
<artifactId>msutil</artifactId> <artifactId>msutil</artifactId>
<version>3.0</version> <version>3.0</version>
<scope>system</scope> <scope>system</scope>
<systemPath>${project.basedir}/lib/msutil.jar</systemPath> <systemPath>${project.basedir}/lib/msutil.jar</systemPath>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.microsoft.sqlserver</groupId> <groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssqlserver</artifactId> <artifactId>mssqlserver</artifactId>
<version>3.0</version> <version>3.0</version>
<scope>system</scope> <scope>system</scope>
<systemPath>${project.basedir}/lib/mssqlserver.jar</systemPath> <systemPath>${project.basedir}/lib/mssqlserver.jar</systemPath>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.pivotal</groupId> <groupId>com.pivotal</groupId>
<artifactId>greenplum-jdbc</artifactId> <artifactId>greenplum-jdbc</artifactId>
<version>5.1.4</version> <version>5.1.4</version>
<scope>system</scope> <scope>system</scope>
<systemPath>${project.basedir}/lib/greenplum-jdbc-5.1.4.jar</systemPath> <systemPath>${project.basedir}/lib/greenplum-jdbc-5.1.4.jar</systemPath>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.dameng</groupId> <groupId>com.dameng</groupId>
<artifactId>dm-jdbc</artifactId> <artifactId>dm-jdbc</artifactId>
<version>1.0.0</version> <version>1.0.0</version>
<scope>system</scope> <scope>system</scope>
<systemPath>${project.basedir}/lib/DmJdbcDriver18.jar</systemPath> <systemPath>${project.basedir}/lib/DmJdbcDriver18.jar</systemPath>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.kingbase</groupId> <groupId>com.kingbase</groupId>
<artifactId>kingbase-jdbc</artifactId> <artifactId>kingbase-jdbc</artifactId>
<version>8.2.0</version> <version>8.2.0</version>
<scope>system</scope> <scope>system</scope>
<systemPath>${project.basedir}/lib/kingbase8-8.2.0.jar</systemPath> <systemPath>${project.basedir}/lib/kingbase8-8.2.0.jar</systemPath>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.mariadb.jdbc</groupId> <groupId>org.mariadb.jdbc</groupId>
<artifactId>mariadb-java-client</artifactId> <artifactId>mariadb-java-client</artifactId>
<scope>runtime</scope> <scope>runtime</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.ibm.db2.jcc</groupId> <groupId>com.ibm.db2.jcc</groupId>
<artifactId>db2jcc</artifactId> <artifactId>db2jcc</artifactId>
<version>db2jcc4</version> <version>db2jcc4</version>
<scope>runtime</scope> <scope>runtime</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.hive</groupId>
<artifactId>commons-lang3</artifactId> <artifactId>hive-jdbc</artifactId>
</dependency> <version>3.1.2</version>
<scope>runtime</scope>
<exclusions>
<exclusion>
<artifactId>jetty-runner</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
</exclusions>
</dependency>
</dependencies> <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
</dependencies>
</project> </project>

3
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/DatabaseFactory.java
View File

@@ -13,6 +13,7 @@ import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.core.database.impl.DatabaseDB2Impl; import com.gitee.dbswitch.core.database.impl.DatabaseDB2Impl;
import com.gitee.dbswitch.core.database.impl.DatabaseDmImpl; import com.gitee.dbswitch.core.database.impl.DatabaseDmImpl;
import com.gitee.dbswitch.core.database.impl.DatabaseGreenplumImpl; import com.gitee.dbswitch.core.database.impl.DatabaseGreenplumImpl;
import com.gitee.dbswitch.core.database.impl.DatabaseHiveImpl;
import com.gitee.dbswitch.core.database.impl.DatabaseKingbaseImpl; import com.gitee.dbswitch.core.database.impl.DatabaseKingbaseImpl;
import com.gitee.dbswitch.core.database.impl.DatabaseMariaDBImpl; import com.gitee.dbswitch.core.database.impl.DatabaseMariaDBImpl;
import com.gitee.dbswitch.core.database.impl.DatabaseMysqlImpl; import com.gitee.dbswitch.core.database.impl.DatabaseMysqlImpl;
@@ -45,6 +46,7 @@ public final class DatabaseFactory {
put(DatabaseTypeEnum.DB2, DatabaseDB2Impl.class.getName()); put(DatabaseTypeEnum.DB2, DatabaseDB2Impl.class.getName());
put(DatabaseTypeEnum.DM, DatabaseDmImpl.class.getName()); put(DatabaseTypeEnum.DM, DatabaseDmImpl.class.getName());
put(DatabaseTypeEnum.KINGBASE, DatabaseKingbaseImpl.class.getName()); put(DatabaseTypeEnum.KINGBASE, DatabaseKingbaseImpl.class.getName());
put(DatabaseTypeEnum.HIVE, DatabaseHiveImpl.class.getName());
} }
}; };
@@ -64,4 +66,5 @@ public final class DatabaseFactory {
private DatabaseFactory() { private DatabaseFactory() {
} }
} }

5
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDB2Impl.java
View File

@@ -10,13 +10,12 @@
package com.gitee.dbswitch.core.database.impl; package com.gitee.dbswitch.core.database.impl;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.core.constant.Const; import com.gitee.dbswitch.common.constant.Const;
import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.AbstractDatabase;
import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.database.IDatabaseInterface;
import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnDescription;
import com.gitee.dbswitch.core.model.ColumnMetaData; import com.gitee.dbswitch.core.model.ColumnMetaData;
import java.util.List; import java.util.List;
import org.springframework.util.CollectionUtils;
/** /**
* 支持DB2数据库的元信息实现 * 支持DB2数据库的元信息实现
@@ -113,7 +112,7 @@ public class DatabaseDB2Impl extends AbstractDatabase implements IDatabaseInterf
retval += ")"; retval += ")";
} }
if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) {
retval += " NOT NULL"; retval += " NOT NULL";
} }

2
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseDmImpl.java
View File

@@ -11,7 +11,7 @@ package com.gitee.dbswitch.core.database.impl;
import java.util.List; import java.util.List;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.core.constant.Const; import com.gitee.dbswitch.common.constant.Const;
import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.AbstractDatabase;
import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.database.IDatabaseInterface;
import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnDescription;

5
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseGreenplumImpl.java
View File

@@ -10,13 +10,12 @@
package com.gitee.dbswitch.core.database.impl; package com.gitee.dbswitch.core.database.impl;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.core.constant.Const; import com.gitee.dbswitch.common.constant.Const;
import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.AbstractDatabase;
import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.database.IDatabaseInterface;
import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnDescription;
import com.gitee.dbswitch.core.model.ColumnMetaData; import com.gitee.dbswitch.core.model.ColumnMetaData;
import java.util.List; import java.util.List;
import org.springframework.util.CollectionUtils;
/** /**
* 支持Greenplum数据库的元信息实现 * 支持Greenplum数据库的元信息实现
@@ -71,7 +70,7 @@ public class DatabaseGreenplumImpl extends AbstractDatabase implements IDatabase
case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_NUMBER:
case ColumnMetaData.TYPE_INTEGER: case ColumnMetaData.TYPE_INTEGER:
case ColumnMetaData.TYPE_BIGNUMBER: case ColumnMetaData.TYPE_BIGNUMBER:
if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) {
if (useAutoInc) { if (useAutoInc) {
retval += "BIGSERIAL"; retval += "BIGSERIAL";
} else { } else {

154
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseHiveImpl.java Normal file
View File

@@ -0,0 +1,154 @@
package com.gitee.dbswitch.core.database.impl;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.common.util.HivePrepareUtils;
import com.gitee.dbswitch.common.constant.Const;
import com.gitee.dbswitch.core.database.AbstractDatabase;
import com.gitee.dbswitch.core.database.IDatabaseInterface;
import com.gitee.dbswitch.core.model.ColumnDescription;
import com.gitee.dbswitch.core.model.ColumnMetaData;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class DatabaseHiveImpl extends AbstractDatabase implements IDatabaseInterface {
public DatabaseHiveImpl() {
super("org.apache.hive.jdbc.HiveDriver");
}
@Override
public List<ColumnDescription> queryTableColumnMeta(String schemaName, String tableName) {
String querySQL = this.getTableFieldsQuerySQL(schemaName, tableName);
List<ColumnDescription> ret = new ArrayList<>();
try (Statement st = connection.createStatement()) {
HivePrepareUtils.prepare(connection, schemaName, tableName);
try (ResultSet rs = st.executeQuery(querySQL)) {
ResultSetMetaData m = rs.getMetaData();
int columns = m.getColumnCount();
for (int i = 1; i <= columns; i++) {
String name = m.getColumnLabel(i);
if (null == name) {
name = m.getColumnName(i);
}
ColumnDescription cd = new ColumnDescription();
cd.setFieldName(name);
cd.setLabelName(name);
cd.setFieldType(m.getColumnType(i));
if (0 != cd.getFieldType()) {
cd.setFieldTypeName(m.getColumnTypeName(i));
cd.setFiledTypeClassName(m.getColumnClassName(i));
cd.setDisplaySize(m.getColumnDisplaySize(i));
cd.setPrecisionSize(m.getPrecision(i));
cd.setScaleSize(m.getScale(i));
cd.setAutoIncrement(m.isAutoIncrement(i));
cd.setNullable(m.isNullable(i) != ResultSetMetaData.columnNoNulls);
} else {
// 处理视图中NULL as fieldName的情况
cd.setFieldTypeName("CHAR");
cd.setFiledTypeClassName(String.class.getName());
cd.setDisplaySize(1);
cd.setPrecisionSize(1);
cd.setScaleSize(0);
cd.setAutoIncrement(false);
cd.setNullable(true);
}
boolean signed = false;
try {
signed = m.isSigned(i);
} catch (Exception ignored) {
// This JDBC Driver doesn't support the isSigned method
// nothing more we can do here by catch the exception.
}
cd.setSigned(signed);
cd.setDbType(DatabaseTypeEnum.HIVE);
ret.add(cd);
}
return ret;
}
} catch (SQLException e) {
throw new RuntimeException(e);
}
}
@Override
public List<String> queryTablePrimaryKeys(String schemaName, String tableName) {
return Collections.emptyList();
}
@Override
public List<ColumnDescription> querySelectSqlColumnMeta(String sql) {
String querySQL = String.format(" %s LIMIT 1", sql.replace(";", ""));
return this.getSelectSqlColumnMeta(querySQL, DatabaseTypeEnum.HIVE);
}
@Override
protected String getTableFieldsQuerySQL(String schemaName, String tableName) {
return String.format("SELECT * FROM `%s`.`%s` ", schemaName, tableName);
}
@Override
protected String getTestQuerySQL(String sql) {
return String.format("explain %s", sql.replace(";", ""));
}
@Override
public String getQuotedSchemaTableCombination(String schemaName, String tableName) {
return String.format(" `%s`.`%s` ", schemaName, tableName);
}
@Override
public String getFieldDefinition(ColumnMetaData v, List<String> pks, boolean useAutoInc,
boolean addCr) {
String fieldname = v.getName();
int type = v.getType();
String retval = " `" + fieldname + "` ";
switch (type) {
case ColumnMetaData.TYPE_TIMESTAMP:
case ColumnMetaData.TYPE_TIME:
retval += "TIMESTAMP";
break;
case ColumnMetaData.TYPE_DATE:
retval += "DATE";
break;
case ColumnMetaData.TYPE_BOOLEAN:
retval += "TINYINT";
break;
case ColumnMetaData.TYPE_NUMBER:
retval += "FLOAT";
break;
case ColumnMetaData.TYPE_INTEGER:
retval += "DECIMAL";
break;
case ColumnMetaData.TYPE_BIGNUMBER:
retval += "BIGINT";
break;
case ColumnMetaData.TYPE_STRING:
retval += "STRING";
break;
case ColumnMetaData.TYPE_BINARY:
retval += "BINARY";
break;
default:
retval += "STRING";
break;
}
if (addCr) {
retval += Const.CR;
}
return retval;
}
}

5
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseKingbaseImpl.java
View File

@@ -10,13 +10,12 @@
package com.gitee.dbswitch.core.database.impl; package com.gitee.dbswitch.core.database.impl;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.core.constant.Const; import com.gitee.dbswitch.common.constant.Const;
import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.AbstractDatabase;
import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.database.IDatabaseInterface;
import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnDescription;
import com.gitee.dbswitch.core.model.ColumnMetaData; import com.gitee.dbswitch.core.model.ColumnMetaData;
import java.util.List; import java.util.List;
import org.springframework.util.CollectionUtils;
/** /**
* 支持Kingbase数据库的元信息实现 * 支持Kingbase数据库的元信息实现
@@ -71,7 +70,7 @@ public class DatabaseKingbaseImpl extends AbstractDatabase implements IDatabaseI
case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_NUMBER:
case ColumnMetaData.TYPE_INTEGER: case ColumnMetaData.TYPE_INTEGER:
case ColumnMetaData.TYPE_BIGNUMBER: case ColumnMetaData.TYPE_BIGNUMBER:
if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) {
if (useAutoInc) { if (useAutoInc) {
retval += "BIGSERIAL"; retval += "BIGSERIAL";
} else { } else {

9
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseMysqlImpl.java
View File

@@ -10,7 +10,7 @@
package com.gitee.dbswitch.core.database.impl; package com.gitee.dbswitch.core.database.impl;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.core.constant.Const; import com.gitee.dbswitch.common.constant.Const;
import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.AbstractDatabase;
import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.database.IDatabaseInterface;
import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnDescription;
@@ -24,7 +24,6 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
/** /**
* 支持MySQL数据库的元信息实现 * 支持MySQL数据库的元信息实现
@@ -131,7 +130,7 @@ public class DatabaseMysqlImpl extends AbstractDatabase implements IDatabaseInte
case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_NUMBER:
case ColumnMetaData.TYPE_INTEGER: case ColumnMetaData.TYPE_INTEGER:
case ColumnMetaData.TYPE_BIGNUMBER: case ColumnMetaData.TYPE_BIGNUMBER:
if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) {
if (useAutoInc) { if (useAutoInc) {
retval += "BIGINT AUTO_INCREMENT NOT NULL"; retval += "BIGINT AUTO_INCREMENT NOT NULL";
} else { } else {
@@ -174,7 +173,7 @@ public class DatabaseMysqlImpl extends AbstractDatabase implements IDatabaseInte
retval += "CHAR(1)"; retval += "CHAR(1)";
} else if (length < 256) { } else if (length < 256) {
retval += "VARCHAR(" + length + ")"; retval += "VARCHAR(" + length + ")";
} else if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { } else if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) {
/* /*
* MySQL5.6中varchar字段为主键时最大长度为254,例如如下的建表语句在MySQL5.7下能通过，但在MySQL5.6下无法通过： * MySQL5.6中varchar字段为主键时最大长度为254,例如如下的建表语句在MySQL5.7下能通过，但在MySQL5.6下无法通过：
* create table `t_test`( * create table `t_test`(
@@ -212,7 +211,7 @@ public class DatabaseMysqlImpl extends AbstractDatabase implements IDatabaseInte
@Override @Override
public String getPrimaryKeyAsString(List<String> pks) { public String getPrimaryKeyAsString(List<String> pks) {
if (!CollectionUtils.isEmpty(pks)) { if (null != pks && !pks.isEmpty()) {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append("`"); sb.append("`");
sb.append(StringUtils.join(pks, "` , `")); sb.append(StringUtils.join(pks, "` , `"));

2
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseOracleImpl.java
View File

@@ -10,7 +10,7 @@
package com.gitee.dbswitch.core.database.impl; package com.gitee.dbswitch.core.database.impl;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.core.constant.Const; import com.gitee.dbswitch.common.constant.Const;
import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.AbstractDatabase;
import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.database.IDatabaseInterface;
import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnDescription;

7
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabasePostgresImpl.java
View File

@@ -10,13 +10,12 @@
package com.gitee.dbswitch.core.database.impl; package com.gitee.dbswitch.core.database.impl;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.core.constant.Const; import com.gitee.dbswitch.common.constant.Const;
import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.AbstractDatabase;
import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.database.IDatabaseInterface;
import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnDescription;
import com.gitee.dbswitch.core.model.ColumnMetaData; import com.gitee.dbswitch.core.model.ColumnMetaData;
import java.util.List; import java.util.List;
import org.springframework.util.CollectionUtils;
/** /**
* 支持PostgreSQL数据库的元信息实现 * 支持PostgreSQL数据库的元信息实现
@@ -71,7 +70,7 @@ public class DatabasePostgresImpl extends AbstractDatabase implements IDatabaseI
case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_NUMBER:
case ColumnMetaData.TYPE_INTEGER: case ColumnMetaData.TYPE_INTEGER:
case ColumnMetaData.TYPE_BIGNUMBER: case ColumnMetaData.TYPE_BIGNUMBER:
if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) {
if (useAutoInc) { if (useAutoInc) {
retval += "BIGSERIAL"; retval += "BIGSERIAL";
} else { } else {
@@ -107,7 +106,7 @@ public class DatabasePostgresImpl extends AbstractDatabase implements IDatabaseI
if (length < 1 || length >= AbstractDatabase.CLOB_LENGTH) { if (length < 1 || length >= AbstractDatabase.CLOB_LENGTH) {
retval += "TEXT"; retval += "TEXT";
} else { } else {
if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) {
retval += "VARCHAR(" + length + ")"; retval += "VARCHAR(" + length + ")";
} else { } else {
retval += "TEXT"; retval += "TEXT";

7
dbswitch-core/src/main/java/com/gitee/dbswitch/core/database/impl/DatabaseSqlserverImpl.java
View File

@@ -10,7 +10,7 @@
package com.gitee.dbswitch.core.database.impl; package com.gitee.dbswitch.core.database.impl;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.core.constant.Const; import com.gitee.dbswitch.common.constant.Const;
import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.AbstractDatabase;
import com.gitee.dbswitch.core.database.IDatabaseInterface; import com.gitee.dbswitch.core.database.IDatabaseInterface;
import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnDescription;
@@ -24,7 +24,6 @@ import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
/** /**
* 支持SQLServer数据库的元信息实现 * 支持SQLServer数据库的元信息实现
@@ -195,7 +194,7 @@ public class DatabaseSqlserverImpl extends AbstractDatabase implements IDatabase
case ColumnMetaData.TYPE_NUMBER: case ColumnMetaData.TYPE_NUMBER:
case ColumnMetaData.TYPE_INTEGER: case ColumnMetaData.TYPE_INTEGER:
case ColumnMetaData.TYPE_BIGNUMBER: case ColumnMetaData.TYPE_BIGNUMBER:
if (!CollectionUtils.isEmpty(pks) && pks.contains(fieldname)) { if (null != pks && !pks.isEmpty() && pks.contains(fieldname)) {
if (useAutoInc) { if (useAutoInc) {
retval += "BIGINT IDENTITY(0,1)"; retval += "BIGINT IDENTITY(0,1)";
} else { } else {
@@ -255,7 +254,7 @@ public class DatabaseSqlserverImpl extends AbstractDatabase implements IDatabase
@Override @Override
public String getPrimaryKeyAsString(List<String> pks) { public String getPrimaryKeyAsString(List<String> pks) {
if (!CollectionUtils.isEmpty(pks)) { if (null != pks && !pks.isEmpty()) {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append("["); sb.append("[");
sb.append(StringUtils.join(pks, "] , [")); sb.append(StringUtils.join(pks, "] , ["));

2
dbswitch-core/src/main/java/com/gitee/dbswitch/core/model/TableDescription.java
View File

@@ -9,7 +9,7 @@
///////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////
package com.gitee.dbswitch.core.model; package com.gitee.dbswitch.core.model;
import com.gitee.dbswitch.core.constant.DBTableType; import com.gitee.dbswitch.common.type.DBTableType;
/** /**
* 数据库表描述符信息定义(Table Description) * 数据库表描述符信息定义(Table Description)

2
dbswitch-core/src/main/java/com/gitee/dbswitch/core/util/GenerateSqlUtils.java
View File

@@ -10,7 +10,7 @@
package com.gitee.dbswitch.core.util; package com.gitee.dbswitch.core.util;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.core.constant.Const; import com.gitee.dbswitch.common.constant.Const;
import com.gitee.dbswitch.core.database.AbstractDatabase; import com.gitee.dbswitch.core.database.AbstractDatabase;
import com.gitee.dbswitch.core.database.DatabaseFactory; import com.gitee.dbswitch.core.database.DatabaseFactory;
import com.gitee.dbswitch.core.model.ColumnDescription; import com.gitee.dbswitch.core.model.ColumnDescription;

8
dbswitch-data/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
</parent> </parent>
<artifactId>dbswitch-data</artifactId> <artifactId>dbswitch-data</artifactId>
@@ -57,6 +57,12 @@
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId> <artifactId>spring-boot-starter-jdbc</artifactId>
<exclusions>
<exclusion>
<artifactId>spring-boot-starter-logging</artifactId>
<groupId>org.springframework.boot</groupId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>

6
dbswitch-data/src/main/java/com/gitee/dbswitch/data/handler/MigrationHandler.java
View File

@@ -204,9 +204,9 @@ public class MigrationHandler implements Supplier<Long> {
.getDatabaseProduceName(sourceDataSource); .getDatabaseProduceName(sourceDataSource);
String fullTableName = CommonUtils.getTableFullNameByDatabase(sourceDatabaseType, String fullTableName = CommonUtils.getTableFullNameByDatabase(sourceDatabaseType,
tableDescription.getSchemaName(), tableDescription.getTableName()); tableDescription.getSchemaName(), tableDescription.getTableName());
Map<String, Integer> columnMetaData = JdbcTemplateUtils Map<String, Integer> columnMetaData = JdbcTemplateUtils.getColumnMetaData(
.getColumnMetaData(new JdbcTemplate(sourceDataSource), sourceDataSource, sourceDatabaseType, tableDescription.getSchemaName(),
fullTableName); tableDescription.getTableName());
List<String> fields = new ArrayList<>(columnMetaData.keySet()); List<String> fields = new ArrayList<>(columnMetaData.keySet());
StatementResultSet srs = sourceOperator StatementResultSet srs = sourceOperator

4
dbswitch-data/src/main/java/com/gitee/dbswitch/data/util/DataSourceUtils.java
View File

@@ -53,6 +53,10 @@ public final class DataSourceUtils {
*/ */
public static HikariDataSource createTargetDataSource( public static HikariDataSource createTargetDataSource(
DbswichProperties.TargetDataSourceProperties description) { DbswichProperties.TargetDataSourceProperties description) {
if (description.getUrl().trim().startsWith("jdbc:hive2://")) {
throw new UnsupportedOperationException("Unsupported hive as target datasource!!!");
}
HikariDataSource ds = new HikariDataSource(); HikariDataSource ds = new HikariDataSource();
ds.setPoolName("The_Target_DB_Connection"); ds.setPoolName("The_Target_DB_Connection");
ds.setJdbcUrl(description.getUrl()); ds.setJdbcUrl(description.getUrl());

48
dbswitch-data/src/main/java/com/gitee/dbswitch/data/util/JdbcTemplateUtils.java
View File

@@ -10,10 +10,13 @@
package com.gitee.dbswitch.data.util; package com.gitee.dbswitch.data.util;
import com.gitee.dbswitch.common.type.DatabaseTypeEnum; import com.gitee.dbswitch.common.type.DatabaseTypeEnum;
import com.gitee.dbswitch.common.util.CommonUtils;
import com.gitee.dbswitch.dbcommon.util.DatabaseAwareUtils; import com.gitee.dbswitch.dbcommon.util.DatabaseAwareUtils;
import com.gitee.dbswitch.common.util.HivePrepareUtils;
import java.sql.Connection; import java.sql.Connection;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.ResultSetMetaData; import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@@ -48,6 +51,8 @@ public final class JdbcTemplateUtils {
return DatabaseTypeEnum.DM; return DatabaseTypeEnum.DM;
} else if (productName.equalsIgnoreCase("Kingbase")) { } else if (productName.equalsIgnoreCase("Kingbase")) {
return DatabaseTypeEnum.KINGBASE; return DatabaseTypeEnum.KINGBASE;
} else if (productName.equalsIgnoreCase("Hive")) {
return DatabaseTypeEnum.HIVE;
} else { } else {
DatabaseDriver databaseDriver = DatabaseDriver.fromProductName(productName); DatabaseDriver databaseDriver = DatabaseDriver.fromProductName(productName);
if (DatabaseDriver.MARIADB == databaseDriver) { if (DatabaseDriver.MARIADB == databaseDriver) {
@@ -71,30 +76,36 @@ public final class JdbcTemplateUtils {
/** /**
* 获取表字段的元信息 * 获取表字段的元信息
* *
* @param sourceJdbcTemplate JdbcTemplate * @param dataSource DataSource
* @param fullTableName 表的全名 * @param databaseType databaseType
* @param schemaName schemaName
* @param tableName tableName
* @return Map<String, Integer> * @return Map<String, Integer>
*/ */
public static Map<String, Integer> getColumnMetaData( public static Map<String, Integer> getColumnMetaData(
JdbcTemplate sourceJdbcTemplate, DataSource dataSource, DatabaseTypeEnum databaseType,
String fullTableName) { String schemaName, String tableName) {
String fullTableName = CommonUtils.getTableFullNameByDatabase(databaseType,
schemaName, tableName);
final String sql = String.format("select * from %s where 1=2", fullTableName); final String sql = String.format("select * from %s where 1=2", fullTableName);
Map<String, Integer> columnMetaData = new HashMap<>(); Map<String, Integer> columnMetaData = new HashMap<>();
sourceJdbcTemplate.execute((Connection connection) -> { try (Connection connection = dataSource.getConnection()) {
try (Statement stmt = connection.createStatement(); try (Statement stmt = connection.createStatement()) {
ResultSet rs = stmt.executeQuery(sql)) { if (connection.getMetaData().getDatabaseProductName().contains("Hive")) {
ResultSetMetaData rsMetaData = rs.getMetaData(); HivePrepareUtils.prepare(connection, schemaName, tableName);
for (int i = 0, len = rsMetaData.getColumnCount(); i < len; i++) { }
columnMetaData.put(rsMetaData.getColumnName(i + 1), rsMetaData.getColumnType(i + 1)); try (ResultSet rs = stmt.executeQuery(sql)) {
ResultSetMetaData rsMetaData = rs.getMetaData();
for (int i = 0, len = rsMetaData.getColumnCount(); i < len; i++) {
columnMetaData.put(rsMetaData.getColumnName(i + 1), rsMetaData.getColumnType(i + 1));
}
return columnMetaData;
} }
return true;
} catch (Exception e) {
throw new RuntimeException(
String.format("获取表:%s 的字段的元信息时失败. 请联系 DBA 核查该库、表信息.", fullTableName), e);
} }
}); } catch (SQLException e) {
throw new RuntimeException(
return columnMetaData; String.format("获取表:%s 的字段的元信息时失败. 请联系 DBA 核查该库、表信息.", fullTableName), e);
}
} }
/** /**
@@ -112,7 +123,8 @@ public final class JdbcTemplateUtils {
String sql = "SELECT count(*) as total FROM information_schema.tables " String sql = "SELECT count(*) as total FROM information_schema.tables "
+ "WHERE table_schema=? AND table_name=? AND ENGINE='InnoDB'"; + "WHERE table_schema=? AND table_name=? AND ENGINE='InnoDB'";
JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource);
return jdbcTemplate.queryForObject(sql, new Object[]{schemaName, tableName}, Integer.class) > 0; return jdbcTemplate.queryForObject(sql, new Object[]{schemaName, tableName}, Integer.class)
> 0;
} }
} }

2
dbswitch-dbchange/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
</parent> </parent>
<artifactId>dbswitch-dbchange</artifactId> <artifactId>dbswitch-dbchange</artifactId>

47
dbswitch-dbcommon/pom.xml
View File

@@ -1,27 +1,40 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
</parent> </parent>
<artifactId>dbswitch-dbcommon</artifactId> <artifactId>dbswitch-dbcommon</artifactId>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId> <artifactId>dbswitch-common</artifactId>
</dependency> <version>${project.version}</version>
<dependency> </dependency>
<groupId>org.apache.commons</groupId> <dependency>
<artifactId>commons-lang3</artifactId> <groupId>org.springframework.boot</groupId>
</dependency> <artifactId>spring-boot-starter-jdbc</artifactId>
<dependency> <exclusions>
<groupId>org.projectlombok</groupId> <exclusion>
<artifactId>lombok</artifactId> <artifactId>spring-boot-starter-logging</artifactId>
<scope>provided</scope> <groupId>org.springframework.boot</groupId>
</dependency> </exclusion>
</dependencies> </exclusions>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
</project> </project>

6
dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/AbstractDatabaseOperator.java
View File

@@ -12,9 +12,9 @@ package com.gitee.dbswitch.dbcommon.database;
import com.gitee.dbswitch.dbcommon.constant.Constants; import com.gitee.dbswitch.dbcommon.constant.Constants;
import com.gitee.dbswitch.dbcommon.domain.StatementResultSet; import com.gitee.dbswitch.dbcommon.domain.StatementResultSet;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement;
import java.util.Objects; import java.util.Objects;
import javax.sql.DataSource; import javax.sql.DataSource;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -94,8 +94,8 @@ public abstract class AbstractDatabaseOperator implements IDatabaseOperator {
log.debug("Execute sql :{}", sql); log.debug("Execute sql :{}", sql);
} }
try (Connection connection = dataSource.getConnection(); try (Connection connection = dataSource.getConnection();
PreparedStatement ps = connection.prepareStatement(sql)) { Statement st = connection.createStatement()) {
return ps.executeUpdate(); return st.executeUpdate(sql);
} catch (SQLException e) { } catch (SQLException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }

4
dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/DatabaseOperatorFactory.java
View File

@@ -12,6 +12,7 @@ package com.gitee.dbswitch.dbcommon.database;
import com.gitee.dbswitch.dbcommon.database.impl.DB2DatabaseOperator; import com.gitee.dbswitch.dbcommon.database.impl.DB2DatabaseOperator;
import com.gitee.dbswitch.dbcommon.database.impl.DmDatabaseOperator; import com.gitee.dbswitch.dbcommon.database.impl.DmDatabaseOperator;
import com.gitee.dbswitch.dbcommon.database.impl.GreenplumDatabaseOperator; import com.gitee.dbswitch.dbcommon.database.impl.GreenplumDatabaseOperator;
import com.gitee.dbswitch.dbcommon.database.impl.HiveDatabaseOperator;
import com.gitee.dbswitch.dbcommon.database.impl.KingbaseDatabaseOperator; import com.gitee.dbswitch.dbcommon.database.impl.KingbaseDatabaseOperator;
import com.gitee.dbswitch.dbcommon.database.impl.MysqlDatabaseOperator; import com.gitee.dbswitch.dbcommon.database.impl.MysqlDatabaseOperator;
import com.gitee.dbswitch.dbcommon.database.impl.OracleDatabaseOperator; import com.gitee.dbswitch.dbcommon.database.impl.OracleDatabaseOperator;
@@ -43,6 +44,7 @@ public final class DatabaseOperatorFactory {
put("DB2", DB2DatabaseOperator::new); put("DB2", DB2DatabaseOperator::new);
put("DM", DmDatabaseOperator::new); put("DM", DmDatabaseOperator::new);
put("KINGBASE", KingbaseDatabaseOperator::new); put("KINGBASE", KingbaseDatabaseOperator::new);
put("HIVE", HiveDatabaseOperator::new);
} }
}; };
@@ -56,7 +58,7 @@ public final class DatabaseOperatorFactory {
String type = DatabaseAwareUtils.getDatabaseNameByDataSource(dataSource).toUpperCase(); String type = DatabaseAwareUtils.getDatabaseNameByDataSource(dataSource).toUpperCase();
if (!DATABASE_OPERATOR_MAPPER.containsKey(type)) { if (!DATABASE_OPERATOR_MAPPER.containsKey(type)) {
throw new RuntimeException( throw new RuntimeException(
String.format("[dbcommon] Unknown Supported database type (%s)", type)); String.format("[dbcommon] Unsupported database type (%s)", type));
} }
return DATABASE_OPERATOR_MAPPER.get(type).apply(dataSource); return DATABASE_OPERATOR_MAPPER.get(type).apply(dataSource);

61
dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/database/impl/HiveDatabaseOperator.java Normal file
View File

@@ -0,0 +1,61 @@
package com.gitee.dbswitch.dbcommon.database.impl;
import com.gitee.dbswitch.common.util.HivePrepareUtils;
import com.gitee.dbswitch.dbcommon.constant.Constants;
import com.gitee.dbswitch.dbcommon.domain.StatementResultSet;
import java.sql.Connection;
import java.sql.ResultSet;
import java.util.List;
import javax.sql.DataSource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@Slf4j
public class HiveDatabaseOperator extends MysqlDatabaseOperator {
public HiveDatabaseOperator(DataSource dataSource) {
super(dataSource);
}
@Override
public StatementResultSet queryTableData(String schemaName, String tableName, List<String> fields,
List<String> orders) {
String sql = String.format("select `%s` from `%s`.`%s` order by `%s` asc ",
StringUtils.join(fields, "`,`"),
schemaName, tableName, StringUtils.join(orders, "`,`"));
return selectHiveTableData(sql, schemaName, tableName);
}
@Override
public StatementResultSet queryTableData(String schemaName, String tableName,
List<String> fields) {
String sql = String.format("select `%s` from `%s`.`%s` ",
StringUtils.join(fields, "`,`"), schemaName, tableName);
return selectHiveTableData(sql, schemaName, tableName);
}
private StatementResultSet selectHiveTableData(String sql, String schemaName, String tableName) {
if (log.isDebugEnabled()) {
log.debug("Query table data sql :{}", sql);
}
try {
Connection connection = dataSource.getConnection();
HivePrepareUtils.prepare(connection, schemaName, tableName);
StatementResultSet srs = new StatementResultSet();
srs.setConnection(connection);
srs.setAutoCommit(srs.getConnection().getAutoCommit());
srs.getConnection().setAutoCommit(false);
srs.setStatement(srs.getConnection()
.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY));
srs.getStatement().setQueryTimeout(Constants.DEFAULT_QUERY_TIMEOUT_SECONDS);
srs.getStatement().setFetchSize(this.fetchSize);
srs.setResultset(srs.getStatement().executeQuery(sql));
return srs;
} catch (Throwable t) {
throw new RuntimeException(t);
}
}
}

2
dbswitch-dbcommon/src/main/java/com/gitee/dbswitch/dbcommon/util/DatabaseAwareUtils.java
View File

@@ -45,6 +45,8 @@ public final class DatabaseAwareUtils {
return "kingbase"; return "kingbase";
} else if (productName.equalsIgnoreCase("DM DBMS")) { } else if (productName.equalsIgnoreCase("DM DBMS")) {
return "dm"; return "dm";
} else if (productName.equalsIgnoreCase("Apache Hive")) {
return "hive";
} }
DatabaseDriver databaseDriver = DatabaseDriver.fromProductName(productName); DatabaseDriver databaseDriver = DatabaseDriver.fromProductName(productName);

2
dbswitch-dbsynch/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
</parent> </parent>
<artifactId>dbswitch-dbsynch</artifactId> <artifactId>dbswitch-dbsynch</artifactId>

2
dbswitch-dbsynch/src/main/java/com/gitee/dbswitch/dbsynch/DatabaseSynchronizeFactory.java
View File

@@ -56,7 +56,7 @@ public final class DatabaseSynchronizeFactory {
String type = DatabaseAwareUtils.getDatabaseNameByDataSource(dataSource).toUpperCase(); String type = DatabaseAwareUtils.getDatabaseNameByDataSource(dataSource).toUpperCase();
if (!DATABASE_SYNC_MAPPER.containsKey(type)) { if (!DATABASE_SYNC_MAPPER.containsKey(type)) {
throw new RuntimeException( throw new RuntimeException(
String.format("[dbsynch] Unknown Supported database type (%s)", type)); String.format("[dbsynch] Unsupported database type (%s)", type));
} }
return DATABASE_SYNC_MAPPER.get(type).apply(dataSource); return DATABASE_SYNC_MAPPER.get(type).apply(dataSource);

8
dbswitch-dbwriter/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
</parent> </parent>
<artifactId>dbswitch-dbwriter</artifactId> <artifactId>dbswitch-dbwriter</artifactId>
@@ -14,6 +14,12 @@
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId> <artifactId>spring-boot-starter-jdbc</artifactId>
<exclusions>
<exclusion>
<artifactId>spring-boot-starter-logging</artifactId>
<groupId>org.springframework.boot</groupId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>

2
dbswitch-dbwriter/src/main/java/com/gitee/dbswitch/dbwriter/DatabaseWriterFactory.java
View File

@@ -66,7 +66,7 @@ public class DatabaseWriterFactory {
if (!DATABASE_WRITER_MAPPER.containsKey(type.trim())) { if (!DATABASE_WRITER_MAPPER.containsKey(type.trim())) {
throw new RuntimeException( throw new RuntimeException(
String.format("[dbwrite] Unknown Supported database type (%s)", type)); String.format("[dbwrite] Unsupported database type (%s)", type));
} }
return DATABASE_WRITER_MAPPER.get(type.trim()).apply(dataSource); return DATABASE_WRITER_MAPPER.get(type.trim()).apply(dataSource);

2
dbswitch-pgwriter/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
</parent> </parent>
<artifactId>dbswitch-pgwriter</artifactId> <artifactId>dbswitch-pgwriter</artifactId>

2
dbswitch-sql/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
</parent> </parent>
<artifactId>dbswitch-sql</artifactId> <artifactId>dbswitch-sql</artifactId>

2
package-tool/pom.xml
View File

@@ -5,7 +5,7 @@
<parent> <parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
</parent> </parent>
<artifactId>package-tool</artifactId> <artifactId>package-tool</artifactId>

27
pom.xml
View File

@@ -2,16 +2,9 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.0.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.gitee.dbswitch</groupId> <groupId>com.gitee.dbswitch</groupId>
<artifactId>dbswitch-parent</artifactId> <artifactId>dbswitch-parent</artifactId>
<version>1.6.3</version> <version>1.6.4</version>
<packaging>pom</packaging> <packaging>pom</packaging>
<name>dbswitch</name> <name>dbswitch</name>
<description>database switch project</description> <description>database switch project</description>
@@ -24,6 +17,7 @@
<maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring-boot.version>2.3.7.RELEASE</spring-boot.version>
</properties> </properties>
<modules> <modules>
@@ -40,12 +34,17 @@
<module>package-tool</module> <module>package-tool</module>
</modules> </modules>
<dependencies> <dependencyManagement>
<dependency> <dependencies>
<groupId>org.springframework.boot</groupId> <dependency>
<artifactId>spring-boot-starter</artifactId> <groupId>org.springframework.boot</groupId>
</dependency> <artifactId>spring-boot-dependencies</artifactId>
</dependencies> <version>${spring-boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build> <build>
<plugins> <plugins>

2
version.cmd
View File

@@ -1,6 +1,6 @@
@echo off @echo off
set APP_VERSION=1.6.3 set APP_VERSION=1.6.4
echo "Clean Project ..." echo "Clean Project ..."
call mvn clean -f pom.xml call mvn clean -f pom.xml