diff --git a/pom.xml b/pom.xml index b4eeecb2e395b6bcc84e4264c63d573858491880..569f63d634a441cb40777656d18f44ab91c35878 100644 --- a/pom.xml +++ b/pom.xml @@ -45,22 +45,6 @@ 3.0.9-SNAPSHOT 11 - - ${revision} - ${revision} - ${revision} - ${revision} - ${revision} - ${revision} - ${revision} - ${revision} - ${revision} - ${revision} - ${revision} - ${revision} - ${revision} - ${revision} - ${revision} 2.1.0 @@ -129,79 +113,84 @@ - com.schbrain.framework - apollo-spring-boot-starter - ${schbrain-apollo.version} + com.schbrain.common + common + ${revision} - com.schbrain.framework - schbrain-base-dao - ${schbrain-base-dao.version} + com.schbrain.common + web-common + ${revision} - com.schbrain.framework - cache-spring-boot-starter - ${schbrain-cache.version} + com.schbrain.common + common-util + ${revision} com.schbrain.common - common - ${schbrain-common.version} + module-tree + ${revision} - com.schbrain.common - common-util - ${schbrain-common-util.version} + com.schbrain.framework + apollo-spring-boot-starter + ${revision} + + + com.schbrain.framework + cache-spring-boot-starter + ${revision} com.schbrain.framework dubbo-spring-boot-starter - ${schbrain-dubbo.version} + ${revision} com.schbrain.framework elasticsearch-spring-boot-starter - ${schbrain-elasticsearch.version} + ${revision} com.schbrain.framework kafka-spring-boot-starter - ${schbrain-kafka.version} + ${revision} com.schbrain.framework logger-spring-boot-starter - ${schbrain-logger.version} - - - com.schbrain.common - module-tree - ${schbrain-module-tree.version} + ${revision} com.schbrain.framework mybatis-spring-boot-starter - ${schbrain-mybatis.version} + ${revision} com.schbrain.framework oss-spring-boot-starter - ${schbrain-oss.version} + ${revision} com.schbrain.framework - schbrain-spring-support - ${schbrain-spring-support.version} + starrocks-spring-boot-starter + ${revision} - com.schbrain.common - web-common - ${schbrain-web-common.version} + com.schbrain.framework + xxl-job-spring-boot-starter + ${revision} com.schbrain.framework - xxl-job-spring-boot-starter - ${schbrain-xxl.version} + schbrain-spring-support + ${revision} + + + com.schbrain.framework + schbrain-base-dao + ${revision} diff --git a/starters/oss-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/oss/util/OssUtils.java b/starters/oss-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/oss/util/OssUtils.java index 5fbf48fbe9be8f512872db4cc51631a53eb9bc4f..4db9896be861685bc787235b250db5c7dbfb64f6 100644 --- a/starters/oss-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/oss/util/OssUtils.java +++ b/starters/oss-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/oss/util/OssUtils.java @@ -203,8 +203,8 @@ public class OssUtils { if (StringUtils.isBlank(domain)) { return ossUrl; } - UrlBuilder originUrlBuilder = UrlBuilder.ofHttp(ossUrl); - return UrlBuilder.ofHttp(domain) + UrlBuilder originUrlBuilder = UrlBuilder.ofHttpWithoutEncode(ossUrl); + return UrlBuilder.ofHttpWithoutEncode(domain) .setPath(originUrlBuilder.getPath()) .setQuery(originUrlBuilder.getQuery()) .build(); diff --git a/starters/pom.xml b/starters/pom.xml index f04c8db75c479a092c26daa721609734fcec70a8..be3943d5da3e170ad205545b35f62dce3b288278 100644 --- a/starters/pom.xml +++ b/starters/pom.xml @@ -24,6 +24,7 @@ mybatis-spring-boot-starter oss-spring-boot-starter xxl-job-spring-boot-starter + starrocks-spring-boot-starter - \ No newline at end of file + diff --git a/starters/starrocks-spring-boot-starter/pom.xml b/starters/starrocks-spring-boot-starter/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..b1550dadbc52dbaefeb8ac7417f6b141ec8acced --- /dev/null +++ b/starters/starrocks-spring-boot-starter/pom.xml @@ -0,0 +1,22 @@ + + + + 4.0.0 + + + com.schbrain.framework + starters + ${revision} + + + starrocks-spring-boot-starter + + + + com.schbrain.framework + mybatis-spring-boot-starter + + + diff --git a/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/StarrocksAutoConfiguration.java b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/StarrocksAutoConfiguration.java new file mode 100644 index 0000000000000000000000000000000000000000..c5e4b3b674b50cddf410df9f6ce6f8ab0f2db701 --- /dev/null +++ b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/StarrocksAutoConfiguration.java @@ -0,0 +1,22 @@ +package com.schbrain.framework.autoconfigure.starrocks; + +import com.schbrain.framework.autoconfigure.starrocks.operation.StarrocksOperationFactory; +import com.schbrain.framework.autoconfigure.starrocks.properties.StarrocksProperties; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; + +/** + * @author liaozan + * @since 2023/11/27 + */ +@AutoConfiguration +@EnableConfigurationProperties(StarrocksProperties.class) +public class StarrocksAutoConfiguration { + + @Bean + public StarrocksOperationFactory starrocksOperationFactory(StarrocksProperties starrocksProperties) { + return new StarrocksOperationFactory(starrocksProperties); + } + +} diff --git a/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksOperationFactory.java b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksOperationFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..b426cf584d765898c74c803ae48d0ee58f7a57a5 --- /dev/null +++ b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksOperationFactory.java @@ -0,0 +1,35 @@ +package com.schbrain.framework.autoconfigure.starrocks.operation; + +import cn.hutool.core.lang.Singleton; +import com.schbrain.framework.autoconfigure.starrocks.properties.StarrocksProperties; + +/** + * @author liaozan + * @since 2023/11/27 + */ +public class StarrocksOperationFactory { + + public static final String STREAM_LOAD_TEMPLATE = "http://%s:%s/api/%s/%s/_stream_load"; + + private static final String CACHE_PREFIX = "starrocks-operation-service-%s"; + + private final StarrocksProperties properties; + + public StarrocksOperationFactory(StarrocksProperties properties) { + this.properties = properties; + } + + public StarrocksService getStarrocksService(String tableName, Class entityClass) { + return getStarrocksService(properties.getDatabase(), tableName, entityClass); + } + + public StarrocksService getStarrocksService(String database, String tableName, Class entityClass) { + return Singleton.get(String.format(CACHE_PREFIX, tableName), () -> new StarrocksServiceImpl<>(createStreamLoadHandler(database, tableName), entityClass)); + } + + private StarrocksStreamLoadHandler createStreamLoadHandler(String database, String tableName) { + String streamLoadUrl = String.format(STREAM_LOAD_TEMPLATE, properties.getHost(), properties.getHttpPort(), database, tableName); + return new StarrocksStreamLoadHandler(streamLoadUrl, properties.getUsername(), properties.getPassword()); + } + +} diff --git a/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksService.java b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksService.java new file mode 100644 index 0000000000000000000000000000000000000000..46928b9f5fc760b7c6997c1f7a60628fcf7d2baa --- /dev/null +++ b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksService.java @@ -0,0 +1,41 @@ +package com.schbrain.framework.autoconfigure.starrocks.operation; + +import java.util.List; + +/** + * @author liaozan + * @since 2023/11/27 + */ +public interface StarrocksService { + + /** + * 单个保存/更新 + */ + void upsert(T entity); + + /** + * 单个保存/更新,传入 columns 只会处理响应的 column + */ + void upsert(T entity, List columns); + + /** + * 批量保存/更新 + */ + void upsertBatch(List entityList); + + /** + * 批量保存/更新,传入 columns 只会处理响应的 column + */ + void upsertBatch(List entityList, List columns); + + /** + * 删除 + */ + void delete(T entity); + + /** + * 批量删除 + */ + void deleteBatch(List entityList); + +} diff --git a/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksServiceImpl.java b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksServiceImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..d746b8322b04fb142e40afc27a7a0717fd3f2114 --- /dev/null +++ b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksServiceImpl.java @@ -0,0 +1,59 @@ +package com.schbrain.framework.autoconfigure.starrocks.operation; + +import com.schbrain.common.util.ValidateUtils; + +import java.util.Collections; +import java.util.List; + +/** + * @author liaozan + * @since 2023/11/27 + */ +public class StarrocksServiceImpl implements StarrocksService { + + // TODO for select + @SuppressWarnings({"FieldCanBeLocal", "unused"}) + private final Class entityClass; + + private final StarrocksStreamLoadHandler handler; + + public StarrocksServiceImpl(StarrocksStreamLoadHandler handler, Class entityClass) { + this.handler = handler; + this.entityClass = entityClass; + } + + @Override + public void upsert(T entity) { + upsert(entity, Collections.emptyList()); + } + + @Override + public void upsert(T entity, List columns) { + ValidateUtils.notNull(entity, "entity不能为空"); + upsertBatch(List.of(entity), columns); + } + + @Override + public void upsertBatch(List entityList) { + upsertBatch(entityList, Collections.emptyList()); + } + + @Override + public void upsertBatch(List entityList, List columns) { + ValidateUtils.notEmpty(entityList, "entityList不能为空"); + handler.upsertBatch(entityList, columns); + } + + @Override + public void delete(T entity) { + ValidateUtils.notNull(entity, "entity不能为空"); + deleteBatch(List.of(entity)); + } + + @Override + public void deleteBatch(List entityList) { + ValidateUtils.notNull(entityList, "entityList不能为空"); + handler.deleteBatch(entityList); + } + +} diff --git a/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksStreamLoadHandler.java b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksStreamLoadHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..284a72061870ae230fa167c8e487c266a54d3bc1 --- /dev/null +++ b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StarrocksStreamLoadHandler.java @@ -0,0 +1,75 @@ +package com.schbrain.framework.autoconfigure.starrocks.operation; + +import cn.hutool.http.HttpRequest; +import com.schbrain.common.exception.ParamInvalidException; +import com.schbrain.common.util.JacksonUtils; +import com.schbrain.common.util.TraceIdUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; +import java.util.Optional; + +/** + * @author liaozan + * @since 2023/11/27 + */ +@Slf4j +public class StarrocksStreamLoadHandler { + + private final String streamLoadUrl; + private final String username; + private final String password; + + public StarrocksStreamLoadHandler(String streamLoadUrl, String username, String password) { + this.streamLoadUrl = streamLoadUrl; + this.username = username; + this.password = password; + } + + public void upsertBatch(List entityList, List columns) { + String content = JacksonUtils.toJsonString(entityList); + String upsertResult = createUpsertRequest(content, columns).execute().body(); + log.debug("Starrocks streamLoad upsert result: {}", upsertResult); + + checkResponse(upsertResult); + } + + public void deleteBatch(List entityList) { + String content = JacksonUtils.toJsonString(entityList); + String deleteResult = createCommonRequest(content).header("columns", "__op='delete'").execute().body(); + log.debug("Starrocks streamLoad delete result: {}", deleteResult); + + checkResponse(deleteResult); + } + + private HttpRequest createUpsertRequest(String content, List columns) { + HttpRequest request = createCommonRequest(content); + if (CollectionUtils.isNotEmpty(columns)) { + request.header("partial_update", Boolean.TRUE.toString()); + request.header("columns", String.join(",", columns)); + } + return request; + } + + private void checkResponse(String result) { + StreamLoadResponse response = JacksonUtils.getObjectFromJson(result, StreamLoadResponse.class); + if (response == null || response.isFailed()) { + throw new ParamInvalidException(Optional.ofNullable(response).map(StreamLoadResponse::getMessage).orElse(result)); + } + } + + private HttpRequest createCommonRequest(String content) { + return HttpRequest.put(streamLoadUrl) + .header("label", TraceIdUtils.get()) + .header("strict_mode", Boolean.TRUE.toString()) + .header("Expect", "100-continue") + .header("format", "json") + .header("strip_outer_array", Boolean.TRUE.toString()) + .header("ignore_json_size", Boolean.TRUE.toString()) + .basicAuth(username, password) + .setFollowRedirects(true) + .body(content); + } + +} diff --git a/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StreamLoadResponse.java b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StreamLoadResponse.java new file mode 100644 index 0000000000000000000000000000000000000000..fde0ae8ed6ddbba61cb3eae41bd79c5908ba2d64 --- /dev/null +++ b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/operation/StreamLoadResponse.java @@ -0,0 +1,68 @@ +package com.schbrain.framework.autoconfigure.starrocks.operation; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +/** + * @author liaozan + * @since 2023/11/27 + */ +@Data +public class StreamLoadResponse { + + private static final String SUCCESS = "Success"; + + @JsonProperty("Label") + public String label; + + @JsonProperty("TxnId") + private Long txnId; + + @JsonProperty("Status") + private String status; + + @JsonProperty("Message") + private String message; + + @JsonProperty("NumberTotalRows") + private Integer numberTotalRows; + + @JsonProperty("NumberLoadedRows") + private Integer numberLoadedRows; + + @JsonProperty("NumberFilteredRows") + private Integer numberFilteredRows; + + @JsonProperty("NumberUnselectedRows") + private Integer numberUnselectedRows; + + @JsonProperty("LoadBytes") + private Integer loadBytes; + + @JsonProperty("LoadTimeMs") + private Integer loadTimeMs; + + @JsonProperty("BeginTxnTimeMs") + private Integer beginTxnTimeMs; + + @JsonProperty("StreamLoadPlanTimeMs") + private Integer streamLoadPlanTimeMs; + + @JsonProperty("ReadDataTimeMs") + private Integer readDataTimeMs; + + @JsonProperty("WriteDataTimeMs") + private Integer writeDataTimeMs; + + @JsonProperty("CommitAndPublishTimeMs") + private Integer commitAndPublishTimeMs; + + public boolean isSuccess() { + return SUCCESS.equals(status); + } + + public boolean isFailed() { + return !isSuccess(); + } + +} diff --git a/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/properties/StarrocksProperties.java b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/properties/StarrocksProperties.java new file mode 100644 index 0000000000000000000000000000000000000000..0880286caad7ab3660cef8de11183288a12f2d76 --- /dev/null +++ b/starters/starrocks-spring-boot-starter/src/main/java/com/schbrain/framework/autoconfigure/starrocks/properties/StarrocksProperties.java @@ -0,0 +1,68 @@ +package com.schbrain.framework.autoconfigure.starrocks.properties; + +import com.mysql.cj.jdbc.Driver; +import com.schbrain.common.util.support.ConfigurableProperties; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + +/** + * @author liaozan + * @since 2023/11/27 + */ +@Data +@Validated +@ConfigurationProperties(prefix = "starrocks") +public class StarrocksProperties implements ConfigurableProperties { + + /** + * 驱动 + */ + @NotBlank + private String driverClassName = Driver.class.getName(); + + /** + * 连接地址 + */ + @NotBlank + private String host; + + /** + * 数据库名 + */ + @NotBlank + private String database; + + /** + * 数据库直连端口 + */ + @NotNull + private Integer port = 9030; + + /** + * http 连接地址 + */ + @NotNull + private Integer httpPort = 8030; + + /** + * 用户名 + */ + @NotBlank + private String username; + + /** + * 密码 + */ + @NotBlank + private String password; + + @Override + public String getNamespace() { + return "starrocks-common"; + } + +} diff --git a/starters/starrocks-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/starters/starrocks-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 0000000000000000000000000000000000000000..5e4ebf5c1e20d7f38d85984b9a7662183da7a6c7 --- /dev/null +++ b/starters/starrocks-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +com.schbrain.framework.autoconfigure.starrocks.StarrocksAutoConfiguration