调整sr写入逻辑

dbswitch-product/dbswitch-product-starrocks/src/main/java/org/dromara/dbswitch/product/sr/FrontendEntity.java

@@ -1,13 +1,22 @@
+// Copyright tang.  All rights reserved.
+// https://gitee.com/inrgihc/dbswitch
+//
+// Use of this source code is governed by a BSD-style license
+//
+// Author: tang (inrgihc@126.com)
+// Date : 2020/1/2
+// Location: beijing , china
+/////////////////////////////////////////////////////////////
 package org.dromara.dbswitch.product.sr;
 
-
 import lombok.Data;
 
 @Data
 public class FrontendEntity {
-    String ip;
+
-    String httpport;
+  private String ip;
-    Boolean alive;
+  private String httpport;
-    Boolean join;
+  private Boolean alive;
-    String role;
+  private Boolean join;
+  private String role;
 }

dbswitch-product/dbswitch-product-starrocks/src/main/java/org/dromara/dbswitch/product/sr/StarRocksUtils.java

@@ -16,7 +16,18 @@ import cn.hutool.db.Entity;
 import cn.hutool.json.JSONArray;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.sql.DataSource;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.http.HttpHeaders;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPut;
@@ -28,31 +39,20 @@ import org.apache.http.impl.client.HttpClients;
 import org.apache.http.util.EntityUtils;
 import org.dromara.dbswitch.common.entity.CloseableDataSource;
 
-import javax.sql.DataSource;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.sql.Timestamp;
-import java.util.*;
-import java.util.stream.Collectors;
-
 @Slf4j
 public final class StarRocksUtils {
 
-
+  private String dbName;
-    private String indexName;
+  private String tbName;
-    private volatile String dbName;
+  private String host;
-    private volatile String tbName;
+  private String username;
-    private volatile String host;
+  private String password;
-    private volatile String username;
+  private CloseableDataSource dataSource;
-    private volatile String password;
+  private String httpPort;
-    private volatile CloseableDataSource dataSource;
-    private volatile String httpPort;
-
 
   public void init(String schemaName, String tableName, DataSource dataSource) {
     this.getHttpPort(dataSource);
     this.dataSource = (CloseableDataSource) dataSource;
-        this.indexName = tableName;
     this.host = ReUtil.extractMulti("jdbc:mysql://(.*):[0-9]{2,8}/", this.dataSource.getJdbcUrl(), "$1");
     this.username = this.dataSource.getUserName();
     this.password = this.dataSource.getPassword();
@@ -65,19 +65,22 @@ public final class StarRocksUtils {
     try {
       List<Entity> frontends = use.query("SHOW FRONTENDS");
       List<FrontendEntity> frontendEntities = BeanUtil.copyToList(frontends, FrontendEntity.class);
-            List<FrontendEntity> leader = frontendEntities.stream().filter(i -> i.getRole().equals("LEADER")).collect(Collectors.toList());
+      List<FrontendEntity> leader = frontendEntities.stream().filter(i -> i.getRole().equals("LEADER"))
+          .collect(Collectors.toList());
       FrontendEntity frontendEntity = leader.get(0);
       this.httpPort = frontendEntity.getHttpport();
     } catch (Exception e) {
-            log.error(e.getMessage());
+      throw new RuntimeException(e);
     }
   }
 
   public long addOrUpdateData(List<String> fieldNames, List<Object[]> recordValues) {
+    if (CollectionUtils.isEmpty(fieldNames) || CollectionUtils.isEmpty(recordValues)) {
+      return 0L;
+    }
     List<Object> objectList = asObjectList(fieldNames, recordValues);
     JSONArray array = JSONUtil.parseArray(objectList);
-        JSONObject jsonObject = JSONUtil.createObj()
+    JSONObject jsonObject = JSONUtil.createObj().set("data", array);
-                .set("data", array);
     try {
       sendData(jsonObject.toString());
       return recordValues.size();
@@ -87,7 +90,6 @@ public final class StarRocksUtils {
   }
 
   private void sendData(String content) throws Exception {
-
     final String loadUrl = String.format("http://%s:%s/api/%s/%s/_stream_load",
         this.host,
         this.httpPort,
@@ -102,7 +104,6 @@ public final class StarRocksUtils {
             return true;
           }
         });
-
     try (CloseableHttpClient client = httpClientBuilder.build()) {
       HttpPut put = new HttpPut(loadUrl);
       StringEntity entity = new StringEntity(content, "UTF-8");
@@ -136,7 +137,6 @@ public final class StarRocksUtils {
     return "Basic " + new String(encoded);
   }
 
-
   private List<Object> asObjectList(List<String> fieldNames, List<Object[]> recordValues) {
     int fieldCount = Math.min(fieldNames.size(), recordValues.get(0).length);
     List<Object> rows = new ArrayList<>(recordValues.size());

dbswitch-product/dbswitch-product-starrocks/src/main/java/org/dromara/dbswitch/product/sr/StarrocksFactoryProvider.java

@@ -9,16 +9,14 @@
 /////////////////////////////////////////////////////////////
 package org.dromara.dbswitch.product.sr;
 
-import org.dromara.dbswitch.core.annotation.Product;
+import javax.sql.DataSource;
 import org.dromara.dbswitch.common.type.ProductTypeEnum;
+import org.dromara.dbswitch.core.annotation.Product;
 import org.dromara.dbswitch.core.features.ProductFeatures;
 import org.dromara.dbswitch.core.provider.AbstractFactoryProvider;
 import org.dromara.dbswitch.core.provider.meta.MetadataProvider;
-import org.dromara.dbswitch.core.provider.sync.AutoCastTableDataSynchronizeProvider;
 import org.dromara.dbswitch.core.provider.sync.TableDataSynchronizeProvider;
-import org.dromara.dbswitch.core.provider.write.AutoCastTableDataWriteProvider;
 import org.dromara.dbswitch.core.provider.write.TableDataWriteProvider;
-import javax.sql.DataSource;
 
 @Product(ProductTypeEnum.STARROCKS)
 public class StarrocksFactoryProvider extends AbstractFactoryProvider {
@@ -47,5 +45,4 @@ public class StarrocksFactoryProvider extends AbstractFactoryProvider {
     return new StarrocksTableDataSynchronizer(this);
   }
 
-
 }

dbswitch-product/dbswitch-product-starrocks/src/main/java/org/dromara/dbswitch/product/sr/StarrocksMetadataQueryProvider.java

@@ -48,6 +48,11 @@ public class StarrocksMetadataQueryProvider extends AbstractMetadataProvider {
   private static final String QUERY_TABLE_METADATA_SQL =
       "SELECT `TABLE_COMMENT`,`TABLE_TYPE` FROM `information_schema`.`TABLES` "
           + "WHERE `TABLE_SCHEMA` = ? AND `TABLE_NAME` = ?";
+  private static final String QUERY_TABLE_PRIMARY_KEY_SQL = " SELECT COLUMN_NAME \n" +
+      " from information_schema.columns\n" +
+      " where TABLE_SCHEMA=? and TABLE_NAME=?\n" +
+      " and TABLE_CATALOG is null\n" +
+      " and COLUMN_KEY = 'PRI'";
 
   public StarrocksMetadataQueryProvider(ProductFactoryProvider factoryProvider) {
     super(factoryProvider);
@@ -166,21 +171,16 @@ public class StarrocksMetadataQueryProvider extends AbstractMetadataProvider {
 
   @Override
   public List<String> queryTablePrimaryKeys(Connection connection, String schemaName, String tableName) {
+    try (PreparedStatement statement = connection.prepareStatement(QUERY_TABLE_PRIMARY_KEY_SQL)) {
+      statement.setString(1, schemaName);
+      statement.setString(2, tableName);
+      try (ResultSet primaryKeys = statement.executeQuery()) {
         List<String> ret = new ArrayList<>();
-    try {
-      Statement statement = connection.createStatement();
-      String sql = String.format(" SELECT * \n" +
-                      " from information_schema.columns\n" +
-                      " where TABLE_SCHEMA=\"%s\" and TABLE_NAME=\"%s\"\n" +
-                      " and TABLE_CATALOG is null\n" +
-                      " and COLUMN_KEY = 'PRI';",
-              schemaName, tableName
-      );
-      ResultSet primaryKeys = statement.executeQuery(sql);
         while (primaryKeys.next()) {
-        ret.add(primaryKeys.getString("COLUMN_NAME"));
+          ret.add(primaryKeys.getString(1));
         }
         return ret.stream().distinct().collect(Collectors.toList());
+      }
     } catch (SQLException e) {
       throw new RuntimeException(e);
     }
@@ -323,7 +323,10 @@ public class StarrocksMetadataQueryProvider extends AbstractMetadataProvider {
         break;
       case ColumnMetaData.TYPE_STRING:
         //see: https://docs.starrocks.io/zh/docs/category/string/
-        if (length <= 65533) {
+        long newLength = length * 3;
+        if (newLength < 255) {
+          retval += "VARCHAR(" + newLength + ")";
+        } else if (newLength <= 65533) {
           retval += "STRING";
         } else if (newLength <= 1048576) {
           retval += "VARCHAR(" + newLength + ")";

dbswitch-product/dbswitch-product-starrocks/src/main/java/org/dromara/dbswitch/product/sr/StarrocksTableDataSynchronizer.java

@@ -9,24 +9,20 @@
 /////////////////////////////////////////////////////////////
 package org.dromara.dbswitch.product.sr;
 
-
+import java.util.List;
-import org.apache.commons.collections4.CollectionUtils;
+import lombok.extern.slf4j.Slf4j;
 import org.dromara.dbswitch.common.entity.CloseableDataSource;
 import org.dromara.dbswitch.core.provider.ProductFactoryProvider;
-import org.dromara.dbswitch.core.provider.sync.DefaultTableDataSynchronizeProvider;
+import org.dromara.dbswitch.core.provider.sync.AutoCastTableDataSynchronizeProvider;
 
-import java.util.ArrayList;
+@Slf4j
-import java.util.List;
+public class StarrocksTableDataSynchronizer extends AutoCastTableDataSynchronizeProvider {
-import java.util.Map;
 
-public class StarrocksTableDataSynchronizer extends DefaultTableDataSynchronizeProvider {
+  private List<String> fieldNames;
-
-    private volatile List<String> fieldNames;
   private final CloseableDataSource dataSource;
 
   private final StarRocksUtils starRocksUtils = new StarRocksUtils();
 
-
   public StarrocksTableDataSynchronizer(ProductFactoryProvider factoryProvider) {
     super(factoryProvider);
     dataSource = (CloseableDataSource) factoryProvider.getDataSource();
@@ -34,75 +30,33 @@ public class StarrocksTableDataSynchronizer extends DefaultTableDataSynchronizeP
 
   @Override
   public void prepare(String schemaName, String tableName, List<String> fieldNames, List<String> pks) {
-        starRocksUtils.init(schemaName, tableName, dataSource);
-
     this.fieldNames = fieldNames;
-
+    super.prepare(schemaName, tableName, fieldNames, pks);
-
+    try {
-        if (fieldNames.isEmpty() || pks.isEmpty() || fieldNames.size() < pks.size()) {
+      starRocksUtils.init(schemaName, tableName, dataSource);
-            throw new IllegalArgumentException("字段列表和主键列表不能为空，或者字段总个数应不小于主键总个数");
+    } catch (Exception e) {
-        }
+      log.warn("Failed to init by StarRocksUtils#init(),information:: {}", e.getMessage());
-        if (!fieldNames.containsAll(pks)) {
-            throw new IllegalArgumentException("字段列表必须包含主键列表");
-        }
-
-        Map<String, Integer> columnType = getTableColumnMetaData(schemaName, tableName, fieldNames);
-        this.fieldOrders = new ArrayList<>(fieldNames);
-        this.pksOrders = new ArrayList<>(pks);
-
-        this.insertStatementSql = getInsertPrepareStatementSql(schemaName, tableName, fieldNames);
-        this.updateStatementSql = getUpdatePrepareStatementSql(schemaName, tableName, fieldNames, pks);
-        this.deleteStatementSql = getDeletePrepareStatementSql(schemaName, tableName, pks);
-
-        insertArgsType = new int[fieldNames.size()];
-        for (int k = 0; k < fieldNames.size(); ++k) {
-            String field = fieldNames.get(k);
-            insertArgsType[k] = columnType.get(field);
-        }
-
-        updateArgsType = new int[fieldNames.size()];
-        int idx = 0;
-        for (int i = 0; i < fieldNames.size(); ++i) {
-            String field = fieldNames.get(i);
-            if (!pks.contains(field)) {
-                updateArgsType[idx++] = columnType.get(field);
     }
   }
-        for (String pk : pks) {
-            updateArgsType[idx++] = columnType.get(pk);
-        }
-
-        deleteArgsType = new int[pks.size()];
-        for (int j = 0; j < pks.size(); ++j) {
-            String pk = pks.get(j);
-            deleteArgsType[j] = columnType.get(pk);
-        }
-    }
-
 
   @Override
   public long executeInsert(List<Object[]> recordValues) {
-        if (CollectionUtils.isEmpty(fieldNames) || CollectionUtils.isEmpty(recordValues)) {
+    try {
-            return 0L;
-        }
-        if (CollectionUtils.isEmpty(fieldNames) || CollectionUtils.isEmpty(recordValues)) {
-            return 0L;
-        }
-
       return starRocksUtils.addOrUpdateData(fieldNames, recordValues);
+    } catch (Exception e) {
+      log.warn("Failed to addOrUpdateData by StarRocksUtils#addOrUpdateData(),information:: {}", e.getMessage());
+      return super.executeInsert(recordValues);
+    }
   }
 
   @Override
   public long executeUpdate(List<Object[]> recordValues) {
-        if (CollectionUtils.isEmpty(fieldNames) || CollectionUtils.isEmpty(recordValues)) {
+    try {
-            return 0L;
-        }
-        if (CollectionUtils.isEmpty(fieldNames) || CollectionUtils.isEmpty(recordValues)) {
-            return 0L;
-        }
-
       return starRocksUtils.addOrUpdateData(fieldNames, recordValues);
+    } catch (Exception e) {
+      log.warn("Failed to addOrUpdateData by StarRocksUtils#addOrUpdateData(),information:: {}", e.getMessage());
+      return super.executeUpdate(recordValues);
+    }
   }
-
 
 }

dbswitch-product/dbswitch-product-starrocks/src/main/java/org/dromara/dbswitch/product/sr/StarrocksTableDataWriteProvider.java

@@ -9,42 +9,41 @@
 /////////////////////////////////////////////////////////////
 package org.dromara.dbswitch.product.sr;
 
+import java.util.List;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections4.CollectionUtils;
 import org.dromara.dbswitch.common.entity.CloseableDataSource;
 import org.dromara.dbswitch.core.provider.ProductFactoryProvider;
-import org.dromara.dbswitch.core.provider.write.DefaultTableDataWriteProvider;
+import org.dromara.dbswitch.core.provider.write.AutoCastTableDataWriteProvider;
-
-import java.util.List;
 
 @Slf4j
-public class StarrocksTableDataWriteProvider extends DefaultTableDataWriteProvider {
+public class StarrocksTableDataWriteProvider extends AutoCastTableDataWriteProvider {
-
 
   private final CloseableDataSource dataSource;
   private final StarRocksUtils starRocksUtils = new StarRocksUtils();
-    ;
 
   public StarrocksTableDataWriteProvider(ProductFactoryProvider factoryProvider) {
     super(factoryProvider);
     dataSource = (CloseableDataSource) factoryProvider.getDataSource();
-
   }
 
   @Override
   public void prepareWrite(String schemaName, String tableName, List<String> fieldNames) {
+    super.prepareWrite(schemaName, tableName, fieldNames);
+    try {
       starRocksUtils.init(schemaName, tableName, dataSource);
+    } catch (Exception e) {
+      log.warn("Failed to init by StarRocksUtils#init(),information: {}", e.getMessage());
+    }
   }
 
   @Override
   public long write(List<String> fieldNames, List<Object[]> recordValues) {
-        if (CollectionUtils.isEmpty(fieldNames) || CollectionUtils.isEmpty(recordValues)) {
+    try {
-            return 0L;
-        }
-
       return starRocksUtils.addOrUpdateData(fieldNames, recordValues);
-
+    } catch (Exception e) {
+      log.warn("Failed to insertOrUpdate data by StarRocksUtils#addOrUpdateData(),information: {}", e.getMessage());
+      return super.write(fieldNames, recordValues);
+    }
   }
-    
 
 }