chore: initialize qiming workspace repository

This commit is contained in:
Codex
2026-05-29 14:22:48 +08:00
commit bfd67a0f2c
10750 changed files with 1885711 additions and 0 deletions

View File

@@ -0,0 +1,80 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.xspaceagi</groupId>
<artifactId>app-platform-compose</artifactId>
<version>0.0.1-SNAPSHOT</version>
</parent>
<artifactId>app-platform-compose-domain</artifactId>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>com.xspaceagi</groupId>
<artifactId>app-platform-compose-adapter</artifactId>
</dependency>
<dependency>
<groupId>com.xspaceagi</groupId>
<artifactId>app-platform-compose-sdk</artifactId>
</dependency>
<dependency>
<groupId>com.xspaceagi</groupId>
<artifactId>app-platform-agent-core-adapter</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.2.5</version> <!-- Use same version as failsafe or a recent one -->
<dependencies>
<!-- Ensure Surefire uses the correct JUnit 5 engine -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.9.1</version> <!-- Match the version used in project dependencies -->
</dependency>
</dependencies>
<configuration>
<!-- Optional: You might want to explicitly tell Surefire to only run *Test.java -->
<!-- <includes>
<include>**/*Test.java</include>
</includes>
<excludes>
<exclude>**/*IT.java</exclude>
</excludes> -->
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>3.2.5</version> <!-- Use appropriate version -->
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
<configuration>
<includes>
<include>**/*IT.java</include>
</includes>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@@ -0,0 +1,38 @@
package com.xspaceagi.compose.domain.config;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* Doris 配置属性
*/
@Slf4j
@Data
@ConfigurationProperties(prefix = "spring.datasource.dynamic.datasource.doris.table")
public class DorisProperties {
/**
* 副本数
* 生产环境通常为3开发环境可以为1
*/
private Integer replicationNum; // 默认值为3
/**
* 分桶数BUCKETS
*/
private Integer bucketNum; // 默认值为10
/**
* 重复键配置
* 如果为空,则使用第一个字段作为重复键
*/
private String duplicateKey;
/**
* 分布键配置
* 如果为空,则使用第一个字段作为分布键
*/
private String distributedKey;
}

View File

@@ -0,0 +1,14 @@
package com.xspaceagi.compose.domain.config;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Configuration
@EnableConfigurationProperties(DorisProperties.class)
public class DorisPropertiesConfig {
// 空类,仅用于激活 DorisProperties 的配置绑定
}

View File

@@ -0,0 +1,24 @@
package com.xspaceagi.compose.domain.dto;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.sdk.vo.data.FrontColumnDefineVo;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import java.util.List;
/**
* 获取列定义信息 (内部类用于返回多个值)
*/
@Getter
@Setter
@AllArgsConstructor
public class ColumnDefinitionResult {
private CustomTableDefinitionModel tableModel;
private List<FrontColumnDefineVo> columnDefines;
private List<String> orderedFieldNames;
private List<CustomFieldDefinitionModel> fields;
}

View File

@@ -0,0 +1,28 @@
package com.xspaceagi.compose.domain.dto;
import java.util.Map;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.Setter;
/**
* 新增业务数据的请求,单行数据
*/
@Schema(description = "新增业务数据的请求,单行数据")
@Getter
@Setter
public class CustomAddBusinessRowDataVo {
/**
* 表ID
*/
@Schema(description = "表ID")
private Long tableId;
/**
* 行数据
*/
@Schema(description = "行数据")
private Map<String, Object> rowData;
}

View File

@@ -0,0 +1,28 @@
package com.xspaceagi.compose.domain.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.Setter;
/**
* 删除业务数据的请求,单行数据
*/
@Schema(description = "删除业务数据的请求,单行数据")
@Getter
@Setter
public class CustomDeleteBusinessRowDataVo {
/**
* 表ID
*/
@Schema(description = "表ID")
private Long tableId;
/**
* 行ID
*/
@Schema(description = "行ID")
private Long rowId;
}

View File

@@ -0,0 +1,79 @@
package com.xspaceagi.compose.domain.dto;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.sdk.vo.define.CreateTableDefineVo;
import com.xspaceagi.system.spec.common.UserContext;
import com.xspaceagi.system.spec.enums.YnEnum;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.Setter;
/**
* 新增 空表定义
*/
@Getter
@Setter
public class CustomEmptyTableVo {
/**
* 表名
*/
private String tableName;
/**
* 表描述
*/
private String tableDescription;
/**
* 所属空间ID
*/
private Long spaceId;
@Schema(description = "图标")
private String icon;
/**
* 转换为表定义模型
* @param model
* @return
*/
public static CustomTableDefinitionModel convertToModel(CustomEmptyTableVo model,UserContext userContext) {
// 1. 创建表定义模型
CustomTableDefinitionModel tableModel = new CustomTableDefinitionModel();
tableModel.setTableName(model.getTableName());
tableModel.setTableDescription(model.getTableDescription());
tableModel.setSpaceId(model.getSpaceId());
tableModel.setIcon(model.getIcon());
// 默认启用
tableModel.setStatus(YnEnum.Y.getKey());
//新增时doris表名未确定先设置为空,等入库后,用id拼接生成doris表名
tableModel.setDorisTable("");
tableModel.setCreatorId(userContext.getUserId());
tableModel.setCreatorName(userContext.getUserName());
return tableModel;
}
/**
* 通过api接口,来创建数据表使用, 将表定义模型转换为空表定义模型
* @param model 表定义模型
* @return
*/
public static CustomEmptyTableVo convertToVo(CreateTableDefineVo model) {
CustomEmptyTableVo vo = new CustomEmptyTableVo();
vo.setTableName(model.getTableName());
vo.setTableDescription(model.getTableDescription());
vo.setSpaceId(model.getSpaceId());
vo.setIcon(model.getIcon());
return vo;
}
}

View File

@@ -0,0 +1,34 @@
package com.xspaceagi.compose.domain.dto;
import java.util.Map;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.Setter;
/**
* 修改业务数据的请求,单行数据
*/
@Schema(description = "修改业务数据的请求,单行数据")
@Getter
@Setter
public class CustomUpdateBusinessRowDataVo {
/**
* 表ID
*/
@Schema(description = "表ID")
private Long tableId;
/**
* 行ID
*/
@Schema(description = "行ID")
private Long rowId;
/**
* 行数据
*/
@Schema(description = "行数据")
private Map<String, Object> rowData;
}

View File

@@ -0,0 +1,14 @@
package com.xspaceagi.compose.domain.dto;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import lombok.AllArgsConstructor;
import lombok.Getter;
import java.util.Map;
@Getter
@AllArgsConstructor
public class FieldCreationResult {
private CustomTableDefinitionModel updatedTableModel;
private Map<String, String> newFieldMapping;
}

View File

@@ -0,0 +1,27 @@
package com.xspaceagi.compose.domain.dto;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.Setter;
/**
* 根据操作日志恢复业务数据的请求
*/
@Schema(description = "根据操作日志恢复业务数据的请求")
@Getter
@Setter
public class RestoreBusinessDataByLogVo {
/**
* 操作日志的 extraContent JSON 字符串
*/
@Schema(description = "操作日志的 extraContent JSON 字符串", required = true,
example = "{\"httpParams\":{},\"spelData\":{\"tableId\":123,\"rowId\":456},\"spelExpression\":\"#request\"}")
private String extraContent;
/**
* 是否强制恢复(如果数据已存在是否覆盖)
*/
@Schema(description = "是否强制恢复(如果数据已存在是否覆盖,默认 false")
private Boolean forceRestore = false;
}

View File

@@ -0,0 +1,44 @@
package com.xspaceagi.compose.domain.model;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**
* 查询doris表数据
*/
@Schema(description = "查询doris表数据")
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
@Data
public class CustomDorisDataRequest {
/**
* 表ID
*/
@Schema(description = "表ID", requiredMode = Schema.RequiredMode.REQUIRED)
@JsonPropertyDescription("表ID")
private Long tableId;
/**
* 页码
*/
@Schema(description = "页码", requiredMode = Schema.RequiredMode.REQUIRED)
@JsonPropertyDescription("页码")
private Long pageNo;
/**
* 页大小
*/
@Schema(description = "页大小", requiredMode = Schema.RequiredMode.REQUIRED)
@JsonPropertyDescription("页大小")
private Long pageSize;
}

View File

@@ -0,0 +1,294 @@
package com.xspaceagi.compose.domain.model;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import com.xspaceagi.compose.sdk.enums.DefaultTableFieldEnum;
import com.xspaceagi.compose.sdk.vo.data.FrontColumnDefineVo;
import com.xspaceagi.compose.sdk.vo.define.TableFieldDefineVo;
import com.xspaceagi.compose.sdk.vo.doris.DorisTableFieldVo;
import com.xspaceagi.system.spec.common.UserContext;
import com.xspaceagi.system.spec.enums.YnEnum;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
/**
* 自定义字段定义
*/
@Schema(description = "自定义字段定义")
@Getter
@Setter
@Builder
public class CustomFieldDefinitionModel {
/**
* 主键ID
*/
@Schema(description = "主键ID")
private Long id;
/**
* 是否为系统字段,1:系统字段;-1:非系统字段
*/
@Schema(description = "是否为系统字段,1:系统字段;-1:非系统字段")
private Integer systemFieldFlag;
/**
* 租户ID
*/
@Schema(description = "租户ID")
private Long tenantId;
/**
* 所属空间ID
*/
@Schema(description = "所属空间ID")
private Long spaceId;
/**
* 关联的表ID
*/
@Schema(description = "关联的表ID")
private Long tableId;
/**
* 字段名
*/
@Schema(description = "字段名")
private String fieldName;
/**
* 字段描述
*/
@Schema(description = "字段描述")
private String fieldDescription;
/**
* 字段类型1:String;2:Integer;3:Number;4:Boolean;5:Date
*/
@Schema(description = "字段类型1:String;2:Integer;3:Number;4:Boolean;5:Date")
private Integer fieldType;
/**
* 是否可为空1-可空 -1-非空
*/
@Schema(description = "是否可为空1-可空 -1-非空")
private Integer nullableFlag;
/**
* 默认值
*/
@Schema(description = "默认值")
private String defaultValue;
/**
* 是否唯一1-唯一 -1-非唯一
*/
@Schema(description = "是否唯一1-唯一 -1-非唯一")
private Integer uniqueFlag;
/**
* 是否启用1-启用 -1-禁用
*/
@Schema(description = "是否启用1-启用 -1-禁用")
private Integer enabledFlag;
/**
* 字段顺序
*/
@Schema(description = "字段顺序")
private Integer sortIndex;
/**
* 创建时间
*/
@Schema(description = "创建时间")
private LocalDateTime created;
/**
* 字符串字段长度,可空,比如字符串,可以指定长度使用
*/
@Schema(description = "字符串字段长度,可空,比如字符串,可以指定长度使用")
private Integer fieldStrLen;
/**
* 创建人id
*/
@Schema(description = "创建人id")
private Long creatorId;
/**
* 创建人
*/
@Schema(description = "创建人")
private String creatorName;
/**
* 更新时间
*/
@Schema(description = "更新时间")
private LocalDateTime modified;
/**
* 最后修改人id
*/
@Schema(description = "最后修改人id")
private Long modifiedId;
/**
* 最后修改人
*/
@Schema(description = "最后修改人")
private String modifiedName;
/**
* 转换为表字段定义VO
*
* @param field
* @return
*/
public static TableFieldDefineVo convertToTableFieldDefineVo(CustomFieldDefinitionModel field) {
TableFieldDefineVo fieldVo = TableFieldDefineVo.builder()
.id(field.getId())
.systemFieldFlag(field.getSystemFieldFlag())
.fieldName(field.getFieldName())
.fieldDescription(field.getFieldDescription())
.fieldType(field.getFieldType())
.nullableFlag(field.getNullableFlag())
.uniqueFlag(field.getUniqueFlag())
.enabledFlag(field.getEnabledFlag())
.defaultValue(field.getDefaultValue())
.build();
return fieldVo;
}
/**
* 获取表的系统字段,第一次新增空白表结构的时候使用
*
* @param tableId 表ID
* @param spaceId 空间ID
* @param userContext 用户上下文
* @return 系统字段列表
*/
public static List<CustomFieldDefinitionModel> obatinTableSystemFields(Long tableId, Long spaceId,
UserContext userContext) {
var systemFields = DefaultTableFieldEnum.getAllDefaultFields();
List<CustomFieldDefinitionModel> fieldList = new ArrayList<>();
AtomicInteger sortIndexCounter = new AtomicInteger(0);
for (TableFieldDefineVo field : systemFields) {
CustomFieldDefinitionModel fieldModel = CustomFieldDefinitionModel.builder()
.id(null)
.systemFieldFlag(YnEnum.Y.getKey())
.tableId(tableId)
.fieldName(field.getFieldName())
.fieldDescription(field.getFieldDescription())
.fieldType(field.getFieldType())
.nullableFlag(field.getNullableFlag())
.uniqueFlag(field.getUniqueFlag())
.enabledFlag(field.getEnabledFlag())
.defaultValue(field.getDefaultValue())
.sortIndex(sortIndexCounter.getAndIncrement())
.spaceId(spaceId)
.tenantId(userContext.getTenantId())
.creatorId(userContext.getUserId())
.creatorName(userContext.getUserName())
.build();
fieldList.add(fieldModel);
}
return fieldList;
}
/**
* 转换为给前端用的表字段定义
*
* @param field 自定义字段定义
* @return Excel抬头字段定义VO
*/
public static FrontColumnDefineVo convertToExcelColumnDefineVo(CustomFieldDefinitionModel field) {
FrontColumnDefineVo columnDefineVo = FrontColumnDefineVo.builder()
.systemFieldFlag(field.getSystemFieldFlag() == YnEnum.Y.getKey())
.fieldName(field.getFieldName())
.fieldDescription(field.getFieldDescription())
.fieldType(field.getFieldType())
.nullableFlag(field.getNullableFlag() == YnEnum.Y.getKey())
.defaultValue(field.getDefaultValue())
.uniqueFlag(field.getUniqueFlag() == YnEnum.Y.getKey())
.enabledFlag(field.getEnabledFlag() == YnEnum.Y.getKey())
.sortIndex(field.getSortIndex())
.build();
return columnDefineVo;
}
/**
* 转换为Doris表字段定义
*
* @param field 自定义字段定义
* @return Doris表字段定义
*/
public static DorisTableFieldVo convertToDorisTableFieldVo(CustomFieldDefinitionModel field) {
DorisTableFieldVo fieldVo = DorisTableFieldVo.builder()
.fieldName(field.getFieldName())
.fieldType(field.getFieldType())
.nullable(field.getNullableFlag() == YnEnum.Y.getKey())
.defaultValue(field.getDefaultValue())
.comment(field.getFieldDescription())
.sortIndex(field.getSortIndex())
.build();
return fieldVo;
}
public static CustomFieldDefinitionModel convertToModelForCreate(TableFieldDefineVo field) {
CustomFieldDefinitionModel fieldModel = CustomFieldDefinitionModel.builder()
.id(null)
.systemFieldFlag(field.getSystemFieldFlag())
.fieldName(field.getFieldName())
.fieldDescription(field.getFieldDescription())
.fieldType(field.getFieldType())
.nullableFlag(field.getNullableFlag())
.uniqueFlag(field.getUniqueFlag())
.enabledFlag(field.getEnabledFlag())
.defaultValue(field.getDefaultValue())
.sortIndex(field.getSortIndex())
.spaceId(null)
.tenantId(null)
.creatorId(null)
.creatorName(null)
.modifiedId(null)
.modifiedName(null)
.build();
return fieldModel;
}
public static CustomFieldDefinitionModel createFromExcelHeader(
CustomTableDefinitionModel tableModel,
String headerName,
String newFieldName,
int sortIndex,
UserContext userContext) {
return CustomFieldDefinitionModel.builder()
.fieldName(newFieldName)
.fieldDescription(headerName)
.fieldType(com.xspaceagi.compose.sdk.enums.TableFieldTypeEnum.MEDIUMTEXT.getCode())
.nullableFlag(1)
.enabledFlag(1)
.tableId(tableModel.getId())
.tenantId(tableModel.getTenantId())
.spaceId(tableModel.getSpaceId())
.uniqueFlag(-1)
.systemFieldFlag(-1)
.sortIndex(sortIndex)
.creatorId(userContext.getUserId())
.creatorName(userContext.getUserName())
.build();
}
}

View File

@@ -0,0 +1,184 @@
package com.xspaceagi.compose.domain.model;
import java.time.LocalDateTime;
import java.util.List;
import java.util.stream.Collectors;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import org.springframework.util.CollectionUtils;
import com.xspaceagi.compose.sdk.vo.data.FrontColumnDefineVo;
import com.xspaceagi.compose.sdk.vo.define.TableDefineVo;
import com.xspaceagi.compose.sdk.vo.define.TableFieldDefineVo;
import com.xspaceagi.system.spec.enums.YnEnum;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.Setter;
/**
* 自定义数据表定义
*/
@Schema(description = "自定义数据表定义")
@Getter
@Setter
public class CustomTableDefinitionModel {
/**
* 主键ID
*/
@Schema(description = "主键ID")
private Long id;
/**
* 租户ID
*/
@Schema(description = "租户ID")
private Long tenantId;
/**
* 所属空间ID
*/
@Schema(description = "所属空间ID")
private Long spaceId;
/**
* 图标图片地址
*/
@Schema(description = "图标图片地址")
private String icon;
/**
* 表名
*/
@Schema(description = "表名")
private String tableName;
/**
* 表描述
*/
@Schema(description = "表描述")
private String tableDescription;
/**
* Doris数据库名
*/
@Schema(description = "Doris数据库名")
private String dorisDatabase;
/**
* Doris表名
*/
@Schema(description = "Doris表名")
private String dorisTable;
/**
* 状态1-启用 -1-禁用
*/
@Schema(description = "状态1-启用 -1-禁用")
private Integer status;
/**
* 创建时间
*/
@Schema(description = "创建时间")
private LocalDateTime created;
/**
* 创建人id
*/
@Schema(description = "创建人id")
private Long creatorId;
/**
* 创建人
*/
@Schema(description = "创建人")
private String creatorName;
/**
* 更新时间
*/
@Schema(description = "更新时间")
private LocalDateTime modified;
/**
* 最后修改人id
*/
@Schema(description = "最后修改人id")
private Long modifiedId;
/**
* 最后修改人
*/
@Schema(description = "最后修改人")
private String modifiedName;
@Schema(description = "原始建表DDL")
@JsonPropertyDescription("原始建表DDL")
private String createTableDdl;
/**
* 表字段列表
*/
@Schema(description = "表字段列表")
private List<CustomFieldDefinitionModel> fieldList;
/**
* 转换为表定义VO
*
* @param tableInfo
* @return
*/
public static TableDefineVo convertToTableDefineVo(CustomTableDefinitionModel tableInfo) {
TableDefineVo tableDefineVo = TableDefineVo.builder()
.id(tableInfo.getId())
.tenantId(tableInfo.getTenantId())
.spaceId(tableInfo.getSpaceId())
.icon(tableInfo.getIcon())
.tableName(tableInfo.getTableName())
.tableDescription(tableInfo.getTableDescription())
.dorisDatabase(tableInfo.getDorisDatabase())
.dorisTable(tableInfo.getDorisTable())
.created(tableInfo.getCreated())
.creatorId(tableInfo.getCreatorId())
.creatorName(tableInfo.getCreatorName())
.modified(tableInfo.getModified())
.build();
if (!CollectionUtils.isEmpty(tableInfo.getFieldList())) {
List<TableFieldDefineVo> fieldDefineVos = tableInfo.getFieldList().stream()
.map(CustomFieldDefinitionModel::convertToTableFieldDefineVo)
.collect(Collectors.toList());
tableDefineVo.setFieldList(fieldDefineVos);
}
return tableDefineVo;
}
/**
* 转换为自定义字段定义模型
*
* @param request
* @return
*/
public static CustomFieldDefinitionModel convert2Model(FrontColumnDefineVo request) {
if (request == null) {
return null;
}
CustomFieldDefinitionModel model = CustomFieldDefinitionModel.builder()
.id(request.getId())
.fieldName(request.getFieldName())
.fieldDescription(request.getFieldDescription())
.fieldType(request.getFieldType())
.nullableFlag(request.getNullableFlag() ? YnEnum.Y.getKey() : YnEnum.N.getKey())
.defaultValue(request.getDefaultValue())
.uniqueFlag(request.getUniqueFlag() ? YnEnum.Y.getKey() : YnEnum.N.getKey())
.enabledFlag(request.getEnabledFlag() ? YnEnum.Y.getKey() : YnEnum.N.getKey())
.sortIndex(request.getSortIndex())
.systemFieldFlag(request.getSystemFieldFlag() ? YnEnum.Y.getKey() : YnEnum.N.getKey())
.build();
return model;
}
}

View File

@@ -0,0 +1,29 @@
package com.xspaceagi.compose.domain.model;
import lombok.Data;
/**
* 查询条件模型
*/
@Data
public class QueryCondition {
/**
* 字段名
*/
private String fieldName;
/**
* 操作符
*/
private String operator;
/**
* 字段值
*/
private Object value;
/**
* 连接符AND/OR
*/
private String connector;
}

View File

@@ -0,0 +1,326 @@
package com.xspaceagi.compose.domain.repository;
import com.xspaceagi.compose.sdk.vo.data.ExecuteRawResultVo;
import com.xspaceagi.compose.sdk.vo.doris.DorisTableDefinitionVo;
import java.util.List;
import java.util.Map;
/**
* doris 数据库操作, repository层
*/
public interface ICustomDorisTableRepository {
/**
* 检查表是否存在数据
*
* @param database Doris数据库名
* @param table Doris表名
* @return 是否存在数据
*/
boolean hasData(String database, String table);
/**
* 检查表是否存在
*
* @param database Doris数据库名
* @param table Doris表名
* @return 是否存在
*/
boolean tableExists(String database, String table);
/**
* 执行建表SQL
*
* @param createTableSql 建表SQL
*/
void executeCreateTable(String createTableSql);
/**
* 创建唯一索引
*
* @param database Doris数据库名
* @param table Doris表名
* @param fieldName 字段名
*/
void createUniqueIndex(String database, String table, String fieldName);
/**
* 删除表
*
* @param database Doris数据库名
* @param table Doris表名
*/
void dropTable(String database, String table);
/**
* 清空表数据
*
* @param database Doris数据库名
* @param table Doris表名
*/
void truncateTable(String database, String table);
/**
* 检查字段值是否唯一
*
* @param database Doris数据库名
* @param table Doris表名
* @param fieldName 字段名
* @param fieldValue 字段值
* @return 是否唯一
*/
boolean isFieldValueUnique(String database, String table, String fieldName, String fieldValue);
/**
* 执行表结构变更SQL
*
* @param database Doris数据库名
* @param table Doris表名
* @param alterSql ALTER TABLE SQL语句
*/
void executeAlterTable(String database, String table, String alterSql);
/**
* 删除表数据
*
* @param database Doris数据库名
* @param table Doris表名
* @param id 主键ID
* @return 影响的行数
*/
int deleteTableDataById(String database, String table, Long id);
/**
* 获取表定义信息
*
* @param database Doris数据库名
* @param table Doris表名
* @return 表定义信息
*/
DorisTableDefinitionVo getTableDefinition(String database, String table);
/**
* 插入单行数据
*
* @param database Doris数据库名
* @param table Doris表名
* @param data 数据Mapkey为字段名value为字段值
* @return 影响的行数
*/
int insertRow(String database, String table, Map<String, Object> data);
/**
* 批量插入数据
*
* @param database Doris数据库名
* @param table Doris表名
* @param dataList 数据列表每个Map代表一行数据
* @return 影响的行数
*/
int batchInsert(String database, String table, List<Map<String, Object>> dataList);
/**
* 更新单行数据
*
* @param database Doris数据库名
* @param table Doris表名
* @param id 主键ID
* @param data 要更新的数据key为字段名value为字段值
* @return 影响的行数
*/
int updateRow(String database, String table, Long id, Map<String, Object> data);
/**
* 批量更新数据
*
* @param database Doris数据库名
* @param table Doris表名
* @param dataList 数据列表每个Map必须包含id字段
* @return 影响的行数
*/
int batchUpdate(String database, String table, List<Map<String, Object>> dataList);
/**
* 根据条件查询数据
*
* @param database Doris数据库名
* @param table Doris表名
* @param conditions 查询条件key为字段名value为字段值
* @param orderBy 排序字段 (可选),格式:字段名 ASC/DESC。为空或null则不排序。
* @param offset 偏移量
* @param limit 限制条数
* @return 查询结果列表
*/
List<Map<String, Object>> queryByConditions(String database, String table,
Map<String, Object> conditions, String orderBy, Integer offset, Integer limit);
/**
* 根据条件统计数据
*
* @param database Doris数据库名
* @param table Doris表名
* @param conditions 查询条件key为字段名value为字段值
* @return 统计结果
*/
long countByConditions(String database, String table, Map<String, Object> conditions);
/**
* 根据条件删除数据
*
* @param database Doris数据库名
* @param table Doris表名
* @param conditions 删除条件key为字段名value为字段值
* @return 影响的行数
*/
int deleteByConditions(String database, String table, Map<String, Object> conditions);
/**
* 批量删除数据
*
* @param database Doris数据库名
* @param table Doris表名
* @param ids ID列表
* @return 影响的行数
*/
int batchDelete(String database, String table, List<Long> ids);
/**
* 查询所有数据(带限制条数)
*
* @param database Doris数据库名
* @param table Doris表名
* @param conditions 查询条件key为字段名value为字段值
* @param orderBy 排序字段 (可选),格式:字段名 ASC/DESC。为空或null则不排序。
* @param limit 限制条数 (为 null 或 < 1 时不限制,由实现层控制最大值)
* @return 查询结果列表
*/
List<Map<String, Object>> queryAllData(String database, String table,
Map<String, Object> conditions, String orderBy, Integer limit);
/**
* 执行用户提供的原始只读SQL查询
*
* @param sql 完整的 SQL 查询语句
* @param params SQL 语句中的参数 (按 '?' 顺序传入)
* @return 查询结果列表
*/
ExecuteRawResultVo executeRawQuery(String sql, Object... params);
/**
* 查询用户的sql语句,返回总条数,sql一定是总条数
* @param sql
* @return
*/
Long countRawQuery(String sql);
/**
* 获取表的建表DDL语句
*
* @param database Doris数据库名
* @param table Doris表名
* @return 建表DDL语句
*/
String getCreateTableDdl(String database, String table);
/**
* 执行原始的管理/DML/DDL SQL语句 (高风险操作,请谨慎使用!)
* <p>
* 注意: 主要用于 DML (INSERT, UPDATE, DELETE) 和 DDL (CREATE, ALTER, DROP) 语句。
* 对于 DML返回受影响的行数。
* 对于 DDL通常返回 0。
* 不应用于 SELECT 语句 (会抛出异常)。
* </p>
*
* @param sql 待执行的SQL语句
* @return 受影响的行数 (对于DML语句) 或 0 (对于DDL语句)
*/
int executeRawAdminSql(String sql);
/**
* 批量执行原始的管理/DML/DDL SQL语句 (高风险操作,请谨慎使用!)
*
* @param sqls 待执行的SQL语句列表
*/
void executeRawAdminSqls(List<String> sqls);
/**
* 批量执行SQL的策略枚举
*/
enum BatchExecuteStrategy {
FAIL_FAST, // 遇到错误立即失败并回滚(原有行为)
CONTINUE_ON_ERROR, // 遇到错误继续执行其他SQL
ATOMIC_BATCH // 原子性批处理,要么全成功要么全失败
}
/**
* 根据不同策略批量执行SQL语句 (高风险操作,请谨慎使用!)
*
* @param sqls SQL语句列表
* @param strategy 执行策略
*/
void executeRawAdminSqlsWithStrategy(List<String> sqls, BatchExecuteStrategy strategy);
/**
* 策略1遇到错误立即失败原有行为
*
* @param sqls SQL语句列表
*/
void executeFailFast(List<String> sqls);
/**
* 策略2遇到错误继续执行其他SQL无事务保护
*
* @param sqls SQL语句列表
*/
void executeContinueOnError(List<String> sqls);
/**
* 策略3原子性批处理逐个执行但保持事务完整性
*
* @param sqls SQL语句列表
*/
void executeAtomicBatch(List<String> sqls);
/**
* 查询单个表的数据总条数
*
* @param database Doris数据库名
* @param table Doris表名
* @return 表的总条数
*/
Long getTableTotal(String database, String table);
/**
* 根据主键id,查询对应数据是否存在
*
* @param database Doris数据库名
* @param table Doris表名
* @param id 主键id
*/
boolean queryDorisTableRowDataById(String database, String table, Long id);
/**
* 清空业务数据
*
* @param database Doris数据库名
* @param table Doris表名
*/
void clearBusinessData(String database, String table);
/**
* 查询是否存在业务数据
*
* @param database Doris数据库名
* @param table Doris表名
* @return
*/
boolean existTableData(String database, String table);
/**
* 执行原始的DDL SQL语句 (高风险操作,请谨慎使用!)
*
* @param sql 待执行的DDL SQL语句
*/
void executeRawDDL(String sql);
}

View File

@@ -0,0 +1,71 @@
package com.xspaceagi.compose.domain.repository;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.system.spec.common.UserContext;
import java.util.List;
public interface ICustomFieldDefinitionRepository {
/**
* 根据ID集合查询列表
*
* @param ids id集合
* @return 列表
*/
List<CustomFieldDefinitionModel> queryListByIds(List<Long> ids);
/**
* 根据主键查询 单条记录
*
* @param id id
* @return 单条记录
*/
CustomFieldDefinitionModel queryOneInfoById(Long id);
/**
* 批量新增
*
* @param entityList 新增列表
* @param userContext 用户上下文
*/
void batchAddInfo(List<CustomFieldDefinitionModel> modelList, UserContext userContext);
/**
* 批量更新
*
* @param entityList 更新列表
* @param userContext 用户上下文
*/
void batchUpdateInfo(List<CustomFieldDefinitionModel> modelList, UserContext userContext);
/**
* 删除根据主键id
*
* @param id id
*/
void deleteById(Long id, UserContext userContext);
/**
* 根据表id删除
*
* @param tableId
*/
void deleteByTableId(Long tableId);
/**
* 根据表id查询列表
*
* @param tableId
* @return
*/
List<CustomFieldDefinitionModel> queryListByTableId(Long tableId);
/**
* 根据表id集合查询列表
*
* @param tableIds
* @return
*/
List<CustomFieldDefinitionModel> queryListByTableIds(List<Long> tableIds);
}

View File

@@ -0,0 +1,116 @@
package com.xspaceagi.compose.domain.repository;
import com.xspaceagi.compose.domain.dto.ColumnDefinitionResult;
import com.xspaceagi.compose.domain.dto.CustomEmptyTableVo;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.system.spec.common.UserContext;
import com.xspaceagi.system.infra.service.IQueryViewService;
import java.util.List;
public interface ICustomTableDefinitionRepository extends IQueryViewService<CustomTableDefinitionModel> {
/**
* 根据ID集合查询列表
*
* @param ids id集合
* @return 列表
*/
List<CustomTableDefinitionModel> queryListByIds(List<Long> ids);
/**
* 根据主键查询 单条记录
*
* @param id id
* @return 单条记录
*/
CustomTableDefinitionModel queryOneInfoById(Long id);
/**
* 更新
*
* @param entity entity
* @return id
*/
Long updateInfo(CustomTableDefinitionModel entity, UserContext userContext);
/**
* 更新表名称
*
* @param entity
* @return
*/
Long updateTableName(CustomTableDefinitionModel entity, UserContext userContext);
/**
* 新增
*/
Long addInfo(CustomTableDefinitionModel entity);
/**
* 新增空表定义
*
* @param vo 空表定义
* @param userContext 用户上下文
* @return
*/
Long addEmptyTableInfo(CustomEmptyTableVo vo, UserContext userContext);
/**
* 删除根据主键id
*
* @param id id
*/
void deleteById(Long id);
/**
* 根据条件查询表定义列表
*
* @param condition 查询条件
* @return 表定义列表
*/
List<CustomTableDefinitionModel> queryListByCondition(CustomTableDefinitionModel condition);
/**
* 根据空间ID查询表定义列表
*
* @param spaceId 空间ID
* @return 表定义列表
*/
List<CustomTableDefinitionModel> queryListBySpaceId(Long spaceId);
/**
* 根据表ID获取列定义
* @param tableId
* @return
*/
public ColumnDefinitionResult getColumnDefinitions(Long tableId);
/**
* 复制表结构定义
*
* @param tableId
* @param userContext
* @return
*/
Long copyTableDefinition(Long tableId, UserContext userContext);
/**
* 修改表定义的最后更新时间
*
* @param tableId
* @param userContext
*/
void updateTableLastModified(Long tableId, UserContext userContext);
/**
* 统计数据表总数
*
* @return 数据表总数
*/
Long countTotalTables(Long userId);
}

View File

@@ -0,0 +1,263 @@
package com.xspaceagi.compose.domain.service;
import com.xspaceagi.compose.domain.dto.ColumnDefinitionResult;
import com.xspaceagi.compose.domain.dto.CustomAddBusinessRowDataVo;
import com.xspaceagi.compose.domain.dto.CustomDeleteBusinessRowDataVo;
import com.xspaceagi.compose.domain.dto.CustomUpdateBusinessRowDataVo;
import com.xspaceagi.compose.domain.model.CustomDorisDataRequest;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.sdk.DorisDataPage;
import com.xspaceagi.compose.sdk.request.DorisTableDataRequest;
import com.xspaceagi.compose.sdk.vo.data.ExecuteRawResultVo;
import com.xspaceagi.compose.sdk.vo.doris.DorisTableDefinitionVo;
import com.xspaceagi.system.spec.common.UserContext;
import java.util.List;
import java.util.Map;
/**
* doris 数据库服务接口
*/
public interface CustomDorisTableDomainService {
/**
* 查询doris表数据,分页查询
*
* @param request
* @return
*/
DorisDataPage<List<Object>> queryPageDorisTableData(DorisTableDataRequest request,
ColumnDefinitionResult columnDefResult);
/**
* Web端分页查询Doris表数据
*
* @param request 查询请求参数
* @return 分页数据结果
*/
DorisDataPage<Map<String, Object>> queryPageDorisTableDataForWeb(CustomDorisDataRequest request,
CustomTableDefinitionModel tableModel);
/**
* 删除doris表数据,根据id删除
*
* @param database Doris数据库名
* @param table Doris表名
* @param id 主键id
*/
void deleteDorisTableRowDataById(String database, String table, Long id);
/**
* 检查Doris表是否存在数据
*
* @param database Doris数据库名
* @param table Doris表名
* @return 是否存在数据
*/
boolean hasData(String database, String table);
/**
* 创建Doris表
*
* @param tableModel 表定义
*/
void createTable(CustomTableDefinitionModel tableModel);
/**
* 删除Doris表
*
* @param database Doris数据库名
* @param table Doris表名
*/
void dropTable(String database, String table);
/**
* 重建Doris表
*
* @param tableModel 表定义
*/
void rebuildTable(CustomTableDefinitionModel tableModel);
/**
* 检查字段值是否唯一
*
* @param database Doris数据库名
* @param table Doris表名
* @param fieldName 字段名
* @param fieldValue 字段值
* @return 是否唯一
*/
boolean isFieldValueUnique(String database, String table, String fieldName, String fieldValue);
/**
* 执行Doris表结构变更SQL
*
* @param database Doris数据库名
* @param table Doris表名
* @param alterSqlStatements ALTER TABLE SQL语句列表
*/
void alterTable(String database, String table, List<String> alterSqlStatements);
/**
* 分页查询表数据 (更通用的接口)
*
* @param database Doris数据库名
* @param table Doris表名
* @param conditions 查询条件key为字段名value为字段值
* @param orderBy 排序字段,格式:字段名 ASC/DESC
* @param pageNo 页码 (从1开始)
* @param pageSize 页大小
* @return 分页结果 DorisDataPage<Map<String, Object>>
*/
DorisDataPage<Map<String, Object>> queryPageTableData(String database, String table,
Map<String, Object> conditions, String orderBy, int pageNo, int pageSize);
/**
* 查询所有数据(带限制条数,更通用的接口)
*
* @param database Doris数据库名
* @param table Doris表名
* @param conditions 查询条件key为字段名value为字段值
* @param orderBy 排序字段,格式:字段名 ASC/DESC
* @param limit 限制条数 (为 null 或 < 1 时不限制,由实现层控制最大值)
* @return 查询结果列表 List<Map<String, Object>>
*/
List<Map<String, Object>> queryAllTableData(String database, String table,
Map<String, Object> conditions, String orderBy, Integer limit);
/**
* 生成唯一索引sql
*
* @param database Doris数据库名
* @param table Doris表名
* @param fields 字段列表
*/
void createUniqueIndexes(String database, String table, List<CustomFieldDefinitionModel> fields);
/**
* 在指定数据库中执行原始只读SQL查询 (仅限 SELECT)
*
* @param database 要查询的数据库名称
* @param sql 完整的 SELECT 查询语句
* @return 查询结果列表
*/
ExecuteRawResultVo executeRawQuery(String database, String sql);
/**
* 检查表是否存在
*
* @param database Doris数据库名
* @param table Doris表名
* @return 表是否存在
*/
boolean tableExists(String database, String table);
/**
* 清空表数据
*
* @param database Doris数据库名
* @param table Doris表名
*/
void truncateTable(String database, String table);
/**
* 获取表定义信息
*
* @param database Doris数据库名
* @param table Doris表名
* @return 表定义信息
*/
DorisTableDefinitionVo getTableDefinition(String database, String table);
/**
* 更新表结构定义。
* 根据传入的新字段列表与现有字段列表进行比较,生成并执行 ALTER TABLE 语句。
* 目前支持的操作:添加新字段、修改字段注释。
*
* @param tableId 要更新的表的ID
* @param newFields 最新的字段定义列表
*/
void updateTableStructure(Long tableId, List<CustomFieldDefinitionModel> newFields,
CustomTableDefinitionModel tableModel);
/**
* 根据请求参数生成执行sql
*
* @param request
* @return
*/
String generateExecuteSql(DorisTableDataRequest request, CustomTableDefinitionModel tableModel);
/**
* 新增业务数据
*
* @param request
* @param tableModel
*/
void addBusinessData(CustomAddBusinessRowDataVo request, CustomTableDefinitionModel tableModel,
UserContext userContext);
/**
* 修改业务数据
*
* @param request
* @param tableModel
*/
void updateBusinessData(CustomUpdateBusinessRowDataVo request, CustomTableDefinitionModel tableModel,
UserContext userContext);
/**
* 删除业务数据
*
* @param request
* @param tableModel
*/
void deleteBusinessData(CustomDeleteBusinessRowDataVo request, CustomTableDefinitionModel tableModel,
UserContext userContext);
/**
* 获取表的总条数
*
* @param database Doris数据库名
* @param table Doris表名
* @return 表的总条数
*/
Long getTableTotal(String database, String table);
/**
* 根据主键id,查询对应数据是否存在
*
* @param database Doris数据库名
* @param table Doris表名
* @param id 主键id
*/
boolean queryDorisTableRowDataById(String database, String table, Long id);
/**
* 清空业务数据
*
* @param tableId
* @param userContext
*/
void clearBusinessData(Long tableId, CustomTableDefinitionModel tableModel, UserContext userContext);
/**
* 查询是否存在业务数据
*
* @param database Doris数据库名
* @param table Doris表名
* @return
*/
boolean existTableData(String database, String table);
/**
* 批量插入数据
*
* @param tableDefinitionModel 表定义
* @param excelData 数据
* @param userContext 用户上下文
*/
void batchInsertData(CustomTableDefinitionModel tableDefinitionModel, List<Map<String, Object>> excelData,
UserContext userContext);
}

View File

@@ -0,0 +1,9 @@
package com.xspaceagi.compose.domain.service;
/**
* 字段定义
*/
public interface CustomFieldDefinitionDomainService {
}

View File

@@ -0,0 +1,145 @@
package com.xspaceagi.compose.domain.service;
import java.io.OutputStream;
import java.util.List;
import com.xspaceagi.compose.domain.dto.CustomEmptyTableVo;
import com.xspaceagi.compose.domain.dto.FieldCreationResult;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.sdk.request.DorisToolTableDefineRequest;
import com.xspaceagi.compose.sdk.request.QueryDorisTableDefinePageRequest;
import com.xspaceagi.system.spec.common.UserContext;
import com.xspaceagi.system.spec.page.SuperPage;
/**
* 表定义
*/
public interface CustomTableDefinitionDomainService {
/**
* 新增空表定义
*
* @param model
* @param userContext
* @return
*/
Long addInfo(CustomEmptyTableVo model, UserContext userContext);
/**
* 更新表定义
*
* @param model
* @param userContext
*/
void updateInfo(CustomTableDefinitionModel model, List<CustomFieldDefinitionModel> fieldList, UserContext userContext);
/**
* 事物更新
* @param model
* @param fieldList
* @param userContext
*/
void updateTableDefinitionInTransaction(CustomTableDefinitionModel model,
List<CustomFieldDefinitionModel> fieldList, UserContext userContext);
/**
* 更新表名称
*
* @param model
* @param userContext
*/
void updateTableName(CustomTableDefinitionModel model, UserContext userContext);
/**
* 删除表定义
*
* @param tableId
* @param userContext
*/
CustomTableDefinitionModel deleteById(Long tableId, UserContext userContext);
/**
* 查询表定义信息
*
* @param tableId 表定义ID
* @return
*/
CustomTableDefinitionModel queryOneTableInfoById(Long tableId);
/**
* 查询所有表定义信息,根据筛选条件查询
*
* @param request 筛选条件
* @return
*/
List<CustomTableDefinitionModel> queryAllTableDefineList(DorisToolTableDefineRequest request);
/**
* 根据空间ID查询表定义信息
*
* @param spaceId 空间ID
* @return 表定义信息列表
*/
List<CustomTableDefinitionModel> queryTableDefineBySpaceId(Long spaceId);
/**
* 导出Doris表数据到Excel
*
* @param tableId 表定义ID
* @param outputStream 输出流用于写入Excel数据
* @param userContext 用户上下文
*/
void exportTableDataToExcel(Long tableId, OutputStream outputStream, UserContext userContext);
/**
* 获取表的原始建表DDL语句
* 如果表不存在,则返回null
*
* @param tableId 表ID
* @param tableDefine 表定义
* @return 建表DDL语句
*/
String getCreateTableDdl(Long tableId, CustomTableDefinitionModel tableDefine);
/**
* 根据空间ID查询表定义信息
*
* @param request 查询请求
* @return 表定义信息列表
*/
SuperPage<CustomTableDefinitionModel> queryPageTableDefine(QueryDorisTableDefinePageRequest request);
/**
* 复制表结构定义
*
* @param tableId
* @param userContext
* @return
*/
Long copyTableDefinition(Long tableId, UserContext userContext);
/**
* 修改表定义的最后更新时间
*
* @param tableId
* @param userContext
*/
void updateTableLastModified(Long tableId, UserContext userContext);
/**
* 根据Excel表头创建新字段
*3
* @param tableModel 当前表模型
* @param newFieldsToCreate 要创建的新字段的中文名列表
* @param userContext 用户上下文
* @return 包含更新后表模型和字段映射的结果
*/
FieldCreationResult handleNewFieldsFromExcel(CustomTableDefinitionModel tableModel, List<String> newFieldsToCreate,
UserContext userContext);
/**
* 统计数据表总数
*
* @return 数据表总数
*/
Long countTotalTables();
}

View File

@@ -0,0 +1,942 @@
package com.xspaceagi.compose.domain.service.impl;
import com.alibaba.fastjson2.JSON;
import com.baomidou.dynamic.datasource.annotation.DS;
import com.baomidou.dynamic.datasource.annotation.DSTransactional;
import com.baomidou.dynamic.datasource.tx.DsPropagation;
import com.google.common.collect.Lists;
import com.xspaceagi.compose.domain.dto.ColumnDefinitionResult;
import com.xspaceagi.compose.domain.dto.CustomAddBusinessRowDataVo;
import com.xspaceagi.compose.domain.dto.CustomDeleteBusinessRowDataVo;
import com.xspaceagi.compose.domain.dto.CustomUpdateBusinessRowDataVo;
import com.xspaceagi.compose.domain.model.CustomDorisDataRequest;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.domain.repository.ICustomDorisTableRepository;
import com.xspaceagi.compose.domain.service.CustomDorisTableDomainService;
import com.xspaceagi.compose.domain.service.CustomTableDefinitionDomainService;
import com.xspaceagi.compose.domain.util.SqlParserUtil;
import com.xspaceagi.compose.domain.util.TableDbWrapperUtil;
import com.xspaceagi.compose.domain.util.ExcelDataValidatorUtil;
import com.xspaceagi.compose.sdk.DorisDataPage;
import com.xspaceagi.compose.sdk.enums.DefaultTableFieldEnum;
import com.xspaceagi.compose.sdk.enums.TableFieldTypeEnum;
import com.xspaceagi.compose.sdk.request.DorisTableDataRequest;
import com.xspaceagi.compose.sdk.vo.data.ExecuteRawResultVo;
import com.xspaceagi.compose.sdk.vo.data.FrontColumnDefineVo;
import com.xspaceagi.compose.sdk.vo.doris.DorisTableDefinitionVo;
import com.xspaceagi.compose.spec.constants.DorisConfigContants;
import com.xspaceagi.compose.spec.utils.ComposeExceptionUtils;
import com.xspaceagi.system.spec.common.UserContext;
import com.xspaceagi.system.spec.exception.BizExceptionCodeEnum;
import com.xspaceagi.system.spec.exception.ComposeException;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import java.math.BigDecimal;
import java.util.*;
import java.util.stream.Collectors;
@DS("doris")
@Slf4j
@Service
public class CustomDorisTableDomainServiceImpl implements CustomDorisTableDomainService {
@Resource
private ICustomDorisTableRepository customDorisTableRepository;
@Resource
private TableDbWrapperUtil tableDbWrapperUtil;
@Lazy
@Resource
private CustomDorisTableDomainService self;
@Lazy
@Resource
private CustomTableDefinitionDomainService customTableDefinitionDomainService;
@Override
@DSTransactional(rollbackFor = Exception.class, propagation = DsPropagation.NOT_SUPPORTED)
public boolean hasData(String database, String table) {
// 先判断业务表是否存在,不存在,则任务不存在数据,不然直接执行查询数据有无的sql会报表不存在
var tableExistFlag = customDorisTableRepository.tableExists(database, table);
if (!tableExistFlag) {
return false;
}
return customDorisTableRepository.hasData(database, table);
}
@Override
@DSTransactional(rollbackFor = Exception.class, propagation = DsPropagation.NOT_SUPPORTED)
public void createTable(CustomTableDefinitionModel tableModel) {
List<CustomFieldDefinitionModel> fields = tableModel.getFieldList();
String createTableSql = tableDbWrapperUtil.buildCreateTableSql(tableModel, fields);
log.info("Create business table, sql={}", createTableSql);
try {
// 创建表
customDorisTableRepository.executeCreateTable(createTableSql);
// 创建唯一索引
this.createUniqueIndexes(tableModel.getDorisDatabase(), tableModel.getDorisTable(), fields);
// 新增:为 uid、agent_id、created 字段自动建普通索引
String db = tableModel.getDorisDatabase();
String tb = tableModel.getDorisTable();
List<String> indexFields = List.of(DefaultTableFieldEnum.UID.getFieldName(),
DefaultTableFieldEnum.AGENT_ID.getFieldName(),
DefaultTableFieldEnum.CREATED.getFieldName());
for (String field : indexFields) {
String idxName = "idx_" + field;
String sql = String.format("CREATE INDEX %s ON `%s`.`%s` (`%s`)", idxName, db, tb, field);
try {
customDorisTableRepository.executeRawAdminSql(sql);
} catch (Exception e) {
log.warn("Create index {} failed (exists or missing column): {}", idxName, e.getMessage());
throw ComposeException.build(BizExceptionCodeEnum.composeCreateIndexFailed);
}
}
} catch (Exception e) {
log.error("Create Doris table error", e);
throw ComposeException.build(BizExceptionCodeEnum.composeCreateTableFailed);
}
}
@Override
public void dropTable(String database, String table) {
// 1. 检查表是否存在
var tableExistFlag = customDorisTableRepository.tableExists(database, table);
if (!tableExistFlag) {
log.warn("Table not found: {}", database + "." + table);
return;
}
String dropTableSql = tableDbWrapperUtil.buildDropTableSql(database, table);
log.info("Delete business data, sql={}", dropTableSql);
customDorisTableRepository.executeRawAdminSql(dropTableSql);
}
@Override
@DSTransactional(rollbackFor = Exception.class, propagation = DsPropagation.NOT_SUPPORTED)
public void rebuildTable(CustomTableDefinitionModel tableModel) {
// 1. 删除原表
this.self.dropTable(tableModel.getDorisDatabase(), tableModel.getDorisTable());
// 2. 创建新表
this.self.createTable(tableModel);
}
@Override
public boolean isFieldValueUnique(String database, String table, String fieldName, String fieldValue) {
return customDorisTableRepository.isFieldValueUnique(database, table, fieldName, fieldValue);
}
@Override
public void deleteDorisTableRowDataById(String database, String table, Long id) {
if (id == null) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "删除数据时ID不能为空");
}
int affectedRows = customDorisTableRepository.deleteTableDataById(database, table, id);
if (affectedRows == 0) {
log.warn("Row not found for delete, database: {}, table: {}, id: {}", database, table, id);
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "未找到要删除的数据");
}
log.info("Doris row deleted, database: {}, table: {}, id: {}", database, table, id);
}
@Override
@DSTransactional(rollbackFor = Exception.class, propagation = DsPropagation.NOT_SUPPORTED)
public void alterTable(String database, String table, List<String> alterSqlStatements) {
log.info("Doris schema change start, database: {}, table: {}", database, table);
if (CollectionUtils.isEmpty(alterSqlStatements)) {
log.warn("No Doris schema change SQL to run");
return;
}
try {
for (String sql : alterSqlStatements) {
customDorisTableRepository.executeAlterTable(database, table, sql);
}
log.info("Doris schema change done, database: {}, table: {}", database, table);
} catch (Exception e) {
log.error("Doris schema change error", e);
var errorMessage = ComposeExceptionUtils.getRootErrorMessage(e);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
}
@Override
public DorisDataPage<List<Object>> queryPageDorisTableData(DorisTableDataRequest request,
ColumnDefinitionResult columnDefResult) {
log.debug("Paged Doris query start, request: {}", request);
if (request == null || request.getTableId() == null) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "请求参数或表ID不能为空");
}
// 1. 获取表定义和字段定义
CustomTableDefinitionModel tableModel = columnDefResult.getTableModel();
List<CustomFieldDefinitionModel> fields = columnDefResult.getFields();
// 2. 构建查询SQL和参数
String finalSql;
List<Object> params = new ArrayList<>();
if (StringUtils.hasText(request.getSql())) {
// 如果提供了SQL使用提供的SQL
finalSql = request.getSql();
// 处理SQL参数
if (request.getArgs() != null) {
params.addAll(request.getArgs().values());
}
} else {
// 如果没有提供SQL构建默认的查询SQL
StringBuilder sql = new StringBuilder("SELECT * FROM `")
.append(tableModel.getDorisDatabase())
.append("`.`")
.append(tableModel.getDorisTable())
.append("`");
// 添加扩展参数作为WHERE条件
if (request.getExtArgs() != null && !request.getExtArgs().isEmpty()) {
sql.append(" WHERE ");
boolean first = true;
for (Map.Entry<String, Object> entry : request.getExtArgs().entrySet()) {
if (!first) {
sql.append(" AND ");
}
sql.append("`").append(entry.getKey()).append("` = ?");
params.add(entry.getValue());
first = false;
}
}
finalSql = sql.toString();
}
// 3. 执行查询
try {
ExecuteRawResultVo executeRawResultVo = customDorisTableRepository.executeRawQuery(finalSql,
params.toArray());
List<Map<String, Object>> rawData = executeRawResultVo.getData();
// 4. 转换数据格式
List<List<Object>> data = transformToRowDataVo(rawData, fields);
// 5. 构建返回结果
DorisDataPage<List<Object>> result = new DorisDataPage<>(1, rawData.size(), rawData.size());
result.setRecords(data);
result.setColumnDefines(columnDefResult.getColumnDefines());
log.debug("Paged Doris query done, data size: {}", data.size());
return result;
} catch (ComposeException ce) {
log.error("DML SQL execution error, SQL: {}", finalSql, ce);
throw ce;
} catch (Exception e) {
log.error("Doris query error, sql: {}, params: {}", finalSql, params, e);
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "执行查询失败: " + e.getMessage());
}
}
private List<List<Object>> transformToRowDataVo(List<Map<String, Object>> rawData,
List<CustomFieldDefinitionModel> fields) {
if (CollectionUtils.isEmpty(rawData)) {
return Collections.emptyList();
}
return rawData.stream()
.map(row -> {
List<Object> rowData = new ArrayList<>();
for (CustomFieldDefinitionModel field : fields) {
String fieldName = field.getFieldName();
Object value = row.get(fieldName);
rowData.add(value != null ? value : "");
}
return rowData;
})
.collect(Collectors.toList());
}
@Override
public DorisDataPage<Map<String, Object>> queryPageTableData(String database, String table,
Map<String, Object> conditions, String orderBy, int pageNo, int pageSize) {
// 构建基础SQL
StringBuilder sql = new StringBuilder("SELECT * FROM `")
.append(database)
.append("`.`")
.append(table)
.append("`");
List<Object> params = new ArrayList<>();
// 添加查询条件
if (conditions != null && !conditions.isEmpty()) {
sql.append(" WHERE ");
boolean first = true;
for (Map.Entry<String, Object> entry : conditions.entrySet()) {
if (!first) {
sql.append(" AND ");
}
sql.append("`").append(entry.getKey()).append("` = ?");
params.add(entry.getValue());
first = false;
}
}
// 添加排序
if (StringUtils.hasText(orderBy)) {
sql.append(" ORDER BY ").append(orderBy);
}
// 添加分页
sql.append(" LIMIT ?, ?");
params.add((pageNo - 1) * pageSize);
params.add(pageSize);
// 执行查询
var executedSql = sql.toString();
ExecuteRawResultVo result = customDorisTableRepository.executeRawQuery(executedSql,
params.toArray());
List<Map<String, Object>> records = result.getData();
// 构建分页结果
DorisDataPage<Map<String, Object>> page = new DorisDataPage<>(pageNo, pageSize);
page.setRecords(records);
return page;
}
@Override
public List<Map<String, Object>> queryAllTableData(String database, String table,
Map<String, Object> conditions, String orderBy, Integer limit) {
log.debug("Doris query all start, database: {}, table: {}, limit: {}, conditions: {}",
database, table, limit, conditions);
// 直接调用 Repository 层的方法,由其处理默认 limit
List<Map<String, Object>> result = customDorisTableRepository.queryAllData(database, table, conditions, orderBy,
limit);
log.debug("Doris query all done, data size: {}", result.size());
return result;
}
/**
* 为唯一字段创建索引
*/
@Override
public void createUniqueIndexes(String database, String table, List<CustomFieldDefinitionModel> fields) {
fields.stream()
.filter(field -> field.getUniqueFlag() != null && field.getUniqueFlag() == 1)
.forEach(field -> customDorisTableRepository.createUniqueIndex(database, table, field.getFieldName()));
}
@Override
public ExecuteRawResultVo executeRawQuery(String database, String sql) {
log.info("Native SQL query start, db: {}, SQL: {}", database, sql);
try {
ExecuteRawResultVo result = customDorisTableRepository.executeRawQuery(sql);
log.info("Native SQL done, {} rows", result.getRowNum());
return result;
} catch (ComposeException ce) {
log.error("DML SQL execution error", ce);
throw ce;
} catch (Exception e) {
log.error("Native SQL failed, db: {}, SQL: {}", database, sql, e);
var errorMessage = ComposeExceptionUtils.getRootErrorMessage(e);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
}
@Override
public boolean tableExists(String database, String table) {
log.info("Check table exists, database: {}, table: {}", database, table);
try {
return customDorisTableRepository.tableExists(database, table);
} catch (Exception e) {
log.error("Check table exists error, database: {}, table: {}", database, table, e);
var errorMessage = ComposeExceptionUtils.getRootErrorMessage(e);
throw ComposeException.build(BizExceptionCodeEnum.composeCheckTableExistsFailed, errorMessage);
}
}
@Override
public void truncateTable(String database, String table) {
log.info("Truncate table start, database: {}, table: {}", database, table);
try {
customDorisTableRepository.truncateTable(database, table);
log.info("Truncate done, database: {}, table: {}", database, table);
} catch (Exception e) {
log.error("Truncate error, database: {}, table: {}", database, table, e);
var errorMessage = ComposeExceptionUtils.getRootErrorMessage(e);
throw ComposeException.build(BizExceptionCodeEnum.composeTruncateTableFailed, errorMessage);
}
}
@Override
public DorisTableDefinitionVo getTableDefinition(String database, String table) {
log.info("Get table definition start, database: {}, table: {}", database, table);
try {
// 检查表是否存在
if (!tableExists(database, table)) {
log.error("Table not found, database: {}, table: {}", database, table);
throw ComposeException.build(BizExceptionCodeEnum.tableNotFound);
}
// 获取表定义信息
DorisTableDefinitionVo definition = customDorisTableRepository.getTableDefinition(database, table);
log.info("Get table definition done, database: {}, table: {}", database, table);
return definition;
} catch (Exception e) {
log.error("Get table definition error, database: {}, table: {}", database, table, e);
var errorMessage = ComposeExceptionUtils.getRootErrorMessage(e);
throw ComposeException.build(BizExceptionCodeEnum.composeGetTableDefinitionFailed, errorMessage);
}
}
@Override
@DSTransactional(rollbackFor = Exception.class)
public void updateTableStructure(Long tableId, List<CustomFieldDefinitionModel> newFields,
CustomTableDefinitionModel tableModel) {
log.info("Update table schema start, tableId: {}", tableId);
if (tableId == null) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "更新表结构时 tableId 不能为空");
}
if (newFields == null) {
log.warn("newFields null, no schema change, tableId: {}", tableId);
return;
}
// 1. 获取当前表定义
String database = tableModel.getDorisDatabase();
String table = tableModel.getDorisTable();
if (!StringUtils.hasText(database) || !StringUtils.hasText(table)) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "表定义缺少数据库名或表名, tableId: " + tableId);
}
// 2. 获取当前字段列表
List<CustomFieldDefinitionModel> oldFields = tableModel.getFieldList();
// 3. 调用工具类生成 ALTER SQL 语句
List<String> potentialAlterSqls = tableDbWrapperUtil.buildAlterTableSqls(database, table, oldFields, newFields);
// 4. 检查表是否有数据,并根据规则过滤 SQL 语句
List<String> finalAlterSqls = new ArrayList<>();
boolean tableHasData = this.self.hasData(database, table);
log.info("Check table {}.{} has data: {}", database, table, tableHasData);
if (!CollectionUtils.isEmpty(potentialAlterSqls)) {
for (String sql : potentialAlterSqls) {
boolean isDropColumn = sql.toUpperCase().contains(" DROP COLUMN ");
if (isDropColumn && tableHasData) {
log.warn("Drop column blocked: table {}.{} has data, columns: [{}]", database, table, sql);
throw ComposeException.build(
BizExceptionCodeEnum.resourceDataNotFound,
String.format("表 %s.%s 中存在数据,不允许删除列。如需删除列,请先清空表数据", database, table));
} else {
finalAlterSqls.add(sql);
}
}
}
// 5. 执行最终确认的 ALTER SQL 语句
if (!CollectionUtils.isEmpty(finalAlterSqls)) {
log.info("Apply {} confirmed schema changes for {}.{}", finalAlterSqls.size(), database, table);
this.self.alterTable(database, table, finalAlterSqls);
log.info("Schema change applied for {}.{}", database, table);
} else {
if (!CollectionUtils.isEmpty(potentialAlterSqls)) {
log.warn("All schema changes blocked (table has data / drop column) for {}.{}", database, table);
} else {
log.info("No schema change needed for {}.{}", database, table);
}
}
log.warn("TODO: update custom_field_definition fields, tableId: {}", tableId);
}
@Override
public DorisDataPage<Map<String, Object>> queryPageDorisTableDataForWeb(CustomDorisDataRequest request,
CustomTableDefinitionModel tableModel) {
log.debug("Web paged Doris query start, request: {}", request);
if (request == null || request.getTableId() == null) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "请求参数或表ID不能为空");
}
// 1. 获取表定义和字段定义
List<CustomFieldDefinitionModel> fields = tableModel.getFieldList();
if (CollectionUtils.isEmpty(fields)) {
log.error("Table definition has no fields, tableId: {}", request.getTableId());
throw ComposeException.build(BizExceptionCodeEnum.composeTableDefMissingFields);
}
// 2. 构建SQL和参数
StringBuilder sql = new StringBuilder("SELECT * FROM `")
.append(tableModel.getDorisDatabase())
.append("`.`")
.append(tableModel.getDorisTable())
.append("`");
// 3. 计算总数的SQL
StringBuilder countSql = new StringBuilder("SELECT COUNT(*) FROM `")
.append(tableModel.getDorisDatabase())
.append("`.`")
.append(tableModel.getDorisTable())
.append("`");
// 4. 添加分页参数
int pageNo = request.getPageNo() != null && request.getPageNo() > 0 ? request.getPageNo().intValue() : 1;
int pageSize = request.getPageSize() != null && request.getPageSize() > 0 ? request.getPageSize().intValue()
: 10;
int offset = (pageNo - 1) * pageSize;
// 5. 添加排序按id正序
sql.append(" ORDER BY `id` ASC");
// 6. 添加分页
sql.append(" LIMIT ?, ?");
List<Object> params = Arrays.asList(offset, pageSize);
try {
// 执行查询
// 查询总数
var executedCountSql = countSql.toString();
log.debug("Run count SQL: {}", executedCountSql);
Long total = customDorisTableRepository.countRawQuery(executedCountSql);
// 查询数据
var executedSql = sql.toString();
log.debug("Run data SQL: {}", executedSql);
ExecuteRawResultVo executeRawResultVo = customDorisTableRepository.executeRawQuery(executedSql,
params.toArray());
List<Map<String, Object>> rawData = executeRawResultVo.getData();
// 长数值字段处理对可能超过JavaScript安全整数范围的数值进行字符串转换避免前端精度丢失
// JavaScript安全整数范围: -(2^53-1) 到 (2^53-1),即 -9007199254740991 到 9007199254740991
// 创建字段类型映射,用于判断哪些字段需要特殊处理
Map<String, Integer> fieldTypeMap = new HashMap<>();
for (CustomFieldDefinitionModel field : fields) {
fieldTypeMap.put(field.getFieldName(), field.getFieldType());
}
// 对数据进行预处理:转换长数值为字符串
for (Map<String, Object> row : rawData) {
if (row == null) {
continue; // 防御性检查:跳过空行
}
for (Map.Entry<String, Object> entry : row.entrySet()) {
if (entry == null) {
continue; // 防御性检查:跳过空条目
}
String fieldName = entry.getKey();
Object fieldValue = entry.getValue();
// 只处理数值类型的字段
if (fieldValue instanceof Number) {
Integer fieldType = fieldTypeMap.get(fieldName);
// 如果字段类型未找到,记录警告但不影响处理
if (fieldType == null) {
log.debug("Field [{}] not in type map, skip web numeric conversion", fieldName);
continue;
}
// 判断是否需要转换为字符串
if (shouldConvertToStringForWeb(fieldValue, fieldType)) {
row.put(fieldName, String.valueOf(fieldValue));
log.debug("Field [{}] value [{}] stringified to avoid precision loss", fieldName, fieldValue);
}
}
}
}
// === 处理布尔类型字段 ===
if (fields != null && !fields.isEmpty() && rawData != null && !rawData.isEmpty()) {
// 重用之前创建的fieldTypeMap
for (Map<String, Object> row : rawData) {
for (Map.Entry<String, Object> entry : row.entrySet()) {
String fieldName = entry.getKey();
Object value = entry.getValue();
Integer fieldType = fieldTypeMap.get(fieldName);
if (fieldType != null && com.xspaceagi.compose.sdk.enums.TableFieldTypeEnum.BOOLEAN.getCode()
.equals(fieldType)) {
if (value == null) {
// 保持为空
continue;
} else if (value instanceof Number number) {
row.put(fieldName, number.intValue() == 1);
} else if ("1".equals(String.valueOf(value))) {
row.put(fieldName, true);
} else if ("0".equals(String.valueOf(value))) {
row.put(fieldName, false);
}
}
}
}
}
// === 布尔类型处理结束 ===
// 构建返回结果
DorisDataPage<Map<String, Object>> result = new DorisDataPage<>(pageNo, pageSize, total);
result.setRecords(rawData);
// 构建前端用的抬头字段定义
List<FrontColumnDefineVo> columnDefines = fields.stream()
.map(CustomFieldDefinitionModel::convertToExcelColumnDefineVo)
.collect(Collectors.toList());
result.setColumnDefines(columnDefines);
var dataSize = Optional.ofNullable(rawData).map(List::size).orElse(0);
log.debug("Web paged Doris query done, total: {}, data size: {}", total, dataSize);
return result;
} catch (ComposeException ce) {
log.error("DML SQL execution error", ce);
throw ce;
} catch (Exception e) {
log.error("Web paged Doris query error, sql: {}, params: {}", sql, params, e);
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "执行查询失败: " + e.getMessage());
}
}
@Override
public String generateExecuteSql(DorisTableDataRequest request, CustomTableDefinitionModel tableModel) {
if (request == null || request.getTableId() == null) {
throw new IllegalArgumentException("Request parameters or table ID cannot be empty");
}
String sql = request.getSql();
if (!StringUtils.hasText(sql)) {
throw new IllegalArgumentException("SQL statement cannot be empty");
}
try {
// 2. 替换参数
if (request.getArgs() != null && !request.getArgs().isEmpty()) {
for (Map.Entry<String, Object> entry : request.getArgs().entrySet()) {
String rawKey = "${{" + entry.getKey() + "}}";
String normalKey = "{{" + entry.getKey() + "}}";
Object value = entry.getValue();
// 1. 先处理原始字符串
if (sql.contains(rawKey)) {
sql = sql.replace(rawKey, value == null ? "" : value.toString());
}
// 2. 再处理普通参数
if (sql.contains(normalKey)) {
String paramValue = formatValue(value);
sql = sql.replace(normalKey, paramValue);
}
}
}
// 3. 使用SqlParserUtil处理SQL
// 3.1 验证SQL
SqlParserUtil.validateSql(sql);
// 3.2 替换表名和添加额外条件
String finalSql = SqlParserUtil.modifySql(
sql, // 原始SQL
tableModel.getDorisTable(), // 实际表名
request.getExtArgs() // 额外的限制条件
);
log.debug("Final SQL: {}", finalSql);
return finalSql;
} catch (JSQLParserException e) {
log.error("SQL parse failed: sql={}, error={}", sql, e.getMessage(), e);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlParseFailed, e.getMessage());
} catch (Exception e) {
log.error("Build SQL failed: sql={}, error={}", sql, e.getMessage(), e);
var errorMessage = ComposeExceptionUtils.getRootErrorMessage(e);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
}
/**
* 格式化SQL值防止SQL注入
* 优化版本:只移除真正危险的特殊字符,保留换行符等格式字符
*/
private String formatValue(Object value) {
if (value == null) {
return "NULL";
}
if (value instanceof Number) {
return value.toString();
}
if (value instanceof Boolean) {
return ((Boolean) value) ? "1" : "0";
}
// 如果是List类型需要展开成逗号分隔的字符串
if (value instanceof List) {
return ((List<?>) value).stream()
.map(this::formatValue)
.collect(Collectors.joining(","));
}
// 对字符串类型进行特殊处理防止SQL注入
String strValue = value.toString();
// 1. 替换单引号防止SQL注入的核心操作
strValue = strValue.replace("'", "''");
// 2. 只移除真正危险的特殊字符,保留换行符等格式字符
// 保留:\n \r \t (换行符、回车符、制表符)
// 移除:\ " (反斜杠、双引号)- 这些可能被用于SQL注入
strValue = strValue.replace("\\", "\\\\").replace("\"", "\\\"");
return "'" + strValue + "'";
}
@Override
public void addBusinessData(CustomAddBusinessRowDataVo request, CustomTableDefinitionModel tableModel,
UserContext userContext) {
// 根据表定义: tableModel ,生成insert sql
String insertSql = tableDbWrapperUtil.buildInsertSql(tableModel, request.getRowData());
// 执行insert sql
customDorisTableRepository.executeRawAdminSql(insertSql);
}
@Override
public void updateBusinessData(CustomUpdateBusinessRowDataVo request, CustomTableDefinitionModel tableModel,
UserContext userContext) {
// 根据表定义: tableModel ,生成update sql
String updateSql = tableDbWrapperUtil.buildUpdateSql(tableModel, request.getRowData(), request.getRowId());
// 执行update sql
customDorisTableRepository.executeRawAdminSql(updateSql);
}
@Override
public void deleteBusinessData(CustomDeleteBusinessRowDataVo request, CustomTableDefinitionModel tableModel,
UserContext userContext) {
// 根据表定义: tableModel ,生成delete sql
String deleteSql = tableDbWrapperUtil.buildDeleteSql(tableModel, request.getRowId());
// 执行delete sql
customDorisTableRepository.executeRawAdminSql(deleteSql);
}
// @Cacheable(value = "tableTotalCache", key = "#database + '.' + #table")
@Override
public Long getTableTotal(String database, String table) {
return customDorisTableRepository.getTableTotal(database, table);
}
@Override
public boolean queryDorisTableRowDataById(String database, String table, Long id) {
return customDorisTableRepository.queryDorisTableRowDataById(database, table, id);
}
@Override
public void clearBusinessData(Long tableId, CustomTableDefinitionModel tableModel, UserContext userContext) {
// 1. 检查表是否存在
if (!tableExists(tableModel.getDorisDatabase(), tableModel.getDorisTable())) {
log.warn("Table missing, cannot truncate, tableId: {}, database: {}, table: {}", tableId, tableModel.getDorisDatabase(),
tableModel.getDorisTable());
throw ComposeException.build(BizExceptionCodeEnum.tableNotFound);
}
// 2. 清空业务数据
customDorisTableRepository.clearBusinessData(tableModel.getDorisDatabase(), tableModel.getDorisTable());
}
@Override
public boolean existTableData(String database, String table) {
return customDorisTableRepository.existTableData(database, table);
}
@Override
public void batchInsertData(CustomTableDefinitionModel tableDefinitionModel, List<Map<String, Object>> excelData,
UserContext userContext) {
var tableId = tableDefinitionModel.getId();
var database = tableDefinitionModel.getDorisDatabase();
var table = tableDefinitionModel.getDorisTable();
if (!tableExists(database, table)) {
log.warn("Table missing, cannot import, tableId: {}, database: {}, table: {}", tableId, database, table);
throw ComposeException.build(BizExceptionCodeEnum.tableNotFound);
}
Long tableTotal = this.self.getTableTotal(database, table);
int dataSize = excelData.size();
if (tableTotal + dataSize > DorisConfigContants.IMPORT_EXCEL_DATA_MAX_ROWS) {
log.warn("Import would exceed row limit, tableId: {}, currentTotal: {}, importSize: {}",
tableId, tableTotal, dataSize);
throw ComposeException.build(BizExceptionCodeEnum.composeTableRowCountExceeded, DorisConfigContants.IMPORT_EXCEL_DATA_MAX_ROWS);
}
if (dataSize > DorisConfigContants.IMPORT_EXCEL_DATA_MAX_ROWS_CHECK) {
log.warn("Excel rows exceed import limit, tableId: {}, size: {}", tableId, dataSize);
throw ComposeException.build(BizExceptionCodeEnum.composeExcelRowCountExceeded, DorisConfigContants.IMPORT_EXCEL_DATA_MAX_ROWS_CHECK);
}
if (dataSize == 0) {
log.warn("Excel empty, tableId: {}", tableId);
return;
}
// Excel数据校验在插入前进行类型和格式验证
log.info("Excel validation start, tableId: {}, totalRows: {}", tableId, dataSize);
try {
ExcelDataValidatorUtil.validateExcelData(tableDefinitionModel, excelData);
log.info("Excel validation passed, tableId: {}", tableId);
} catch (ComposeException e) {
log.error("Excel validation failed, tableId: {}, error: {}", tableId, e.getMessage());
throw e;
}
// 分批插入数据
List<List<Map<String, Object>>> batches = Lists.partition(excelData,
DorisConfigContants.IMPORT_EXCEL_DATA_BATCH_SIZE);
int batchCount = batches.size();
for (int i = 0; i < batchCount; i++) {
List<Map<String, Object>> batchData = batches.get(i);
List<String> insertSqls = new ArrayList<>();
for (Map<String, Object> originRowData : batchData) {
if(originRowData.isEmpty()){
log.debug("Empty row, skip");
continue;
}
String insertSql = tableDbWrapperUtil.buildInsertSql(tableDefinitionModel, originRowData);
insertSqls.add(insertSql);
}
if (log.isDebugEnabled()) {
log.debug("Batch insert SQL: {}", JSON.toJSONString(insertSqls));
}
customDorisTableRepository.executeRawAdminSqls(insertSqls);
int startIndex = i * DorisConfigContants.IMPORT_EXCEL_DATA_BATCH_SIZE + 1;
int endIndex = Math.min((i + 1) * DorisConfigContants.IMPORT_EXCEL_DATA_BATCH_SIZE, dataSize);
log.info("Imported rows {} to {}, total {}", startIndex, endIndex, endIndex - startIndex + 1);
}
}
/**
* 判断数值是否需要转换为字符串以避免前端JavaScript精度丢失
* JavaScript安全整数范围: -(2^53-1) 到 (2^53-1),即 -9007199254740991 到 9007199254740991
*
* @param fieldValue 字段值
* @param fieldType 字段类型
* @return 是否需要转换为字符串
*/
private boolean shouldConvertToStringForWeb(Object fieldValue, Integer fieldType) {
// 基础检查字段值必须是非null的数值类型
if (fieldValue == null || !(fieldValue instanceof Number)) {
return false;
}
// 字段类型检查如果字段类型为null则不进行转换
if (fieldType == null) {
log.debug("Field type null, skip web numeric conversion");
return false;
}
// 配置检查如果未启用Web端智能转换则不进行处理
if (!DorisConfigContants.ENABLE_WEB_SMART_NUMBER_CONVERSION) {
return false;
}
Number number = (Number) fieldValue;
try {
// 主键类型根据配置决定是否转换为字符串
if (TableFieldTypeEnum.PRIMARY_KEY.getCode().equals(fieldType)) {
return DorisConfigContants.FORCE_CONVERT_PRIMARY_KEY;
}
// INTEGER类型超过JavaScript安全整数范围时转换为字符串
if (TableFieldTypeEnum.INTEGER.getCode().equals(fieldType)) {
long longValue = number.longValue();
return longValue > DorisConfigContants.JS_MAX_SAFE_INTEGER || longValue < DorisConfigContants.JS_MIN_SAFE_INTEGER;
}
// NUMBER类型处理大数值和高精度数值
if (TableFieldTypeEnum.NUMBER.getCode().equals(fieldType)) {
return isUnsafeForJavaScript(fieldValue, number);
}
// 对于所有其他数值类型,也进行基本的安全检查
return isUnsafeForJavaScript(fieldValue, number);
} catch (Exception e) {
// 如果在处理过程中发生任何异常,记录错误但不影响整体流程
log.error("Web numeric conversion error, fieldValue: {}, fieldType: {}", fieldValue, fieldType, e);
return false;
}
}
/**
* 检查数值是否对JavaScript来说是不安全的会丢失精度
*
* @param fieldValue 原始字段值
* @param number 数值对象
* @return 是否不安全需要转换为字符串
*/
private boolean isUnsafeForJavaScript(Object fieldValue, Number number) {
try {
// 方案1检查绝对值是否超过JavaScript安全阈值
double doubleValue = number.doubleValue();
if (Math.abs(doubleValue) >= DorisConfigContants.JS_SAFE_NUMBER_THRESHOLD) {
return true;
}
// 方案2检查是否超过JavaScript安全整数范围
if (fieldValue instanceof Long || fieldValue instanceof Integer) {
long longValue = number.longValue();
if (longValue > DorisConfigContants.JS_MAX_SAFE_INTEGER || longValue < DorisConfigContants.JS_MIN_SAFE_INTEGER) {
return true;
}
}
// 方案3对于BigDecimal检查精度和范围
if (fieldValue instanceof BigDecimal) {
BigDecimal bd = (BigDecimal) fieldValue;
// 检查绝对值是否超过安全阈值
if (bd.abs().compareTo(BigDecimal.valueOf(DorisConfigContants.JS_SAFE_NUMBER_THRESHOLD)) >= 0) {
return true;
}
// 检查有效数字位数
String plainString = bd.toPlainString().replace("-", "").replace(".", "");
// 移除前导零
plainString = plainString.replaceFirst("^0+", "");
if (plainString.length() > DorisConfigContants.JS_MAX_SAFE_DIGITS) {
return true;
}
}
// 方案4检查字符串表示的有效数字位数
String numberStr = String.valueOf(fieldValue);
if (numberStr.contains(".") || numberStr.contains("e") || numberStr.contains("E")) {
// 有小数或科学计数法,需要检查有效数字
String cleanStr = numberStr.replace("-", "").replace(".", "").replace("e", "").replace("E", "");
// 移除前导零
cleanStr = cleanStr.replaceFirst("^0+", "");
if (cleanStr.length() > DorisConfigContants.JS_MAX_SAFE_DIGITS) {
return true;
}
}
return false;
} catch (Exception e) {
log.error("JS number safety check error, fieldValue: {}", fieldValue, e);
// 发生异常时,为了安全起见,建议转换为字符串
return true;
}
}
}

View File

@@ -0,0 +1,18 @@
package com.xspaceagi.compose.domain.service.impl;
import com.xspaceagi.compose.domain.repository.ICustomFieldDefinitionRepository;
import com.xspaceagi.compose.domain.service.CustomFieldDefinitionDomainService;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
@Slf4j
@Service
public class CustomFieldDefinitionDomainServiceImpl implements CustomFieldDefinitionDomainService {
@Resource
private ICustomFieldDefinitionRepository customFieldDefinitionRepository;
}

View File

@@ -0,0 +1,886 @@
package com.xspaceagi.compose.domain.service.impl;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import cn.idev.excel.FastExcel;
import com.alibaba.fastjson2.JSON;
import com.baomidou.dynamic.datasource.annotation.DSTransactional;
import com.baomidou.dynamic.datasource.tx.DsPropagation;
import com.xspaceagi.agent.core.adapter.application.ModelApplicationService;
import com.xspaceagi.compose.domain.dto.CustomEmptyTableVo;
import com.xspaceagi.compose.domain.dto.FieldCreationResult;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.domain.repository.ICustomDorisTableRepository;
import com.xspaceagi.compose.domain.repository.ICustomFieldDefinitionRepository;
import com.xspaceagi.compose.domain.repository.ICustomTableDefinitionRepository;
import com.xspaceagi.compose.domain.service.CustomDorisTableDomainService;
import com.xspaceagi.compose.domain.service.CustomTableDefinitionDomainService;
import com.xspaceagi.compose.domain.util.TableDbWrapperUtil;
import com.xspaceagi.compose.domain.util.excel.ExcelUtils;
import com.xspaceagi.compose.sdk.enums.DefaultTableFieldEnum;
import com.xspaceagi.compose.sdk.enums.TableFieldTypeEnum;
import com.xspaceagi.compose.sdk.request.DorisToolTableDefineRequest;
import com.xspaceagi.compose.sdk.request.QueryDorisTableDefinePageRequest;
import com.xspaceagi.compose.sdk.vo.define.TableFieldDefineVo;
import com.xspaceagi.compose.spec.constants.DorisConfigContants;
import com.xspaceagi.system.infra.redis.annotation.RedisLock;
import com.xspaceagi.system.spec.common.UserContext;
import com.xspaceagi.system.spec.exception.BizExceptionCodeEnum;
import com.xspaceagi.system.spec.exception.ComposeException;
import com.xspaceagi.system.spec.page.SuperPage;
import com.xspaceagi.system.spec.utils.DateUtil;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import java.io.OutputStream;
import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@Service
public class CustomTableDefinitionDomainServiceImpl implements CustomTableDefinitionDomainService {
@Resource
private ICustomTableDefinitionRepository customTableDefinitionRepository;
@Resource
private ICustomFieldDefinitionRepository customFieldDefinitionRepository;
@Resource
private CustomDorisTableDomainService customDorisTableDomainService;
@Resource
private ICustomDorisTableRepository customDorisTableRepository;
@Resource
private TableDbWrapperUtil tableDbWrapperUtil;
@Resource
private ModelApplicationService modelApplicationService;
@Lazy
@Resource
private CustomTableDefinitionDomainService self;
private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
@Override
@DSTransactional(rollbackFor = Exception.class)
public Long addInfo(CustomEmptyTableVo model, UserContext userContext) {
log.debug("Create empty table definition start, params: {}", model);
// 1. 创建表定义模型
CustomTableDefinitionModel tableModel = CustomEmptyTableVo.convertToModel(model, userContext);
// 从配置中获取Doris数据库名称
String dorisDatabase = tableDbWrapperUtil.getDorisDatabase();
tableModel.setDorisDatabase(dorisDatabase);
log.debug("Using Doris database: {}", dorisDatabase);
// 2. 保存表定义
Long tableId = customTableDefinitionRepository.addInfo(tableModel);
// 空表结构添加,新增默认的系统字段
var fieldList = CustomFieldDefinitionModel.obatinTableSystemFields(tableId, model.getSpaceId(), userContext);
customFieldDefinitionRepository.batchAddInfo(fieldList, userContext);
log.debug("Empty table definition created, tableId: {}", tableId);
return tableId;
}
@Override
@RedisLock(prefix = "lock:table_definition:", key = "#model.id", waitTime = 3, leaseTime = 30)
public void updateInfo(CustomTableDefinitionModel model, List<CustomFieldDefinitionModel> fieldList,
UserContext userContext) {
log.debug("Updating table definition, tableId: {}", model.getId());
// 校验所有新增/更新字段的字段名是否合法
if (fieldList != null) {
for (CustomFieldDefinitionModel field : fieldList) {
String fieldName = field.getFieldName();
if (!StringUtils.hasText(fieldName)) {
throw ComposeException.build(BizExceptionCodeEnum.fieldRequiredButEmpty, "字段名");
}
if (fieldName.length() > 64) {
throw ComposeException.build(BizExceptionCodeEnum.composeFieldNameTooLong);
}
if (!Character.isLowerCase(fieldName.charAt(0)) && !Character.isUpperCase(fieldName.charAt(0))) {
throw ComposeException.build(BizExceptionCodeEnum.composeFieldNameMustStartWithLetter);
}
if (!fieldName.matches("^[a-zA-Z][a-zA-Z0-9_]*$")) {
throw ComposeException.build(BizExceptionCodeEnum.composeFieldNameInvalidChars);
}
// 新增:数值型字段的默认值校验
Integer fieldType = field.getFieldType();
String defaultValue = field.getDefaultValue();
if ((TableFieldTypeEnum.INTEGER.getCode().equals(fieldType)
|| TableFieldTypeEnum.NUMBER.getCode().equals(fieldType))
&& StringUtils.hasText(defaultValue)) {
try {
new java.math.BigDecimal(defaultValue);
} catch (Exception e) {
log.warn("Field [{}] default [{}] is not numeric", fieldName, defaultValue);
throw ComposeException.build(BizExceptionCodeEnum.composeDefaultValueMustBeNumber, fieldName);
}
}
if (TableFieldTypeEnum.BOOLEAN.getCode().equals(fieldType) && StringUtils.hasText(defaultValue)) {
if (!(defaultValue.equals("true") || defaultValue.equals("false"))) {
log.warn("Field [{}] default [{}] is not boolean", fieldName, defaultValue);
throw ComposeException.build(BizExceptionCodeEnum.composeDefaultValueMustBeBoolean, fieldName);
}
}
if (TableFieldTypeEnum.STRING.getCode().equals(fieldType) && StringUtils.hasText(defaultValue)) {
if (defaultValue.length() > DorisConfigContants.DEFAULT_STRING_LENGTH) {
log.warn("Field [{}] default [{}] exceeds max length {}", fieldName, defaultValue,
DorisConfigContants.DEFAULT_STRING_LENGTH);
throw ComposeException.build(BizExceptionCodeEnum.composeDefaultValueTooLong, fieldName);
}
}
// if (TableFieldTypeEnum.MEDIUMTEXT.getCode().equals(fieldType) &&
// StringUtils.hasText(defaultValue)) {
// log.warn("Field [{}] is MEDIUMTEXT, default not allowed [{}]", fieldName, defaultValue);
// throw ComposeException.build(BizExceptionCodeEnum.composeMediumtextNoDefault,
// fieldName);
// }
// check 默认值的范围,如果字段类型是 int,范围限制在: [-2147483648,2147483647] 区间
if (field.getFieldType() == TableFieldTypeEnum.INTEGER.getCode()) {
if (StringUtils.hasText(field.getDefaultValue())) {
// 这里通过数值字符串,来判断数值范围,不然字符串数值转 int等类型,如果数值范围过大,会报错
var defaultValueInt = Long.parseLong(defaultValue);
Long minValue = -2147483648L;
Long maxValue = 2147483647L;
if (defaultValueInt < minValue || defaultValueInt > maxValue) {
throw ComposeException.build(BizExceptionCodeEnum.composeDefaultValueOutOfRange, field.getFieldName(),
minValue.toString(), maxValue.toString());
}
}
} else if (field.getFieldType() == TableFieldTypeEnum.NUMBER.getCode()) {
// NUMBER 类型,对应 DECIMAL(28,6),检查默认值范围,如果超出范围,则抛出异常;
if (StringUtils.hasText(field.getDefaultValue())) {
try {
BigDecimal defaultValueBigDecimal = new BigDecimal(field.getDefaultValue());
// DECIMAL(28,6) 的范围是: 整数部分最多22位,小数部分最多6位
// 最大值: 9999999999999999999999.999999
// 最小值: -9999999999999999999999.999999
BigDecimal maxValue = new BigDecimal("9999999999999999999999.999999");
BigDecimal minValue = new BigDecimal("-9999999999999999999999.999999");
if (defaultValueBigDecimal.compareTo(maxValue) > 0
|| defaultValueBigDecimal.compareTo(minValue) < 0) {
throw ComposeException.build(BizExceptionCodeEnum.composeDefaultValueOutOfRange,
field.getFieldName(), minValue.toString(), maxValue.toString());
}
} catch (NumberFormatException e) {
log.warn("Field [{}] default [{}] is not a valid number", field.getFieldName(), field.getDefaultValue());
throw ComposeException.build(BizExceptionCodeEnum.composeDefaultValueMustBeNumber,
field.getFieldName());
}
}
}
}
}
this.self.updateTableDefinitionInTransaction(model, fieldList, userContext);
log.debug("Table definition update done, tableId: {}", model.getId());
}
@Override
@DSTransactional(rollbackFor = Exception.class, propagation = DsPropagation.REQUIRED)
public void updateTableDefinitionInTransaction(CustomTableDefinitionModel model,
List<CustomFieldDefinitionModel> fieldList, UserContext userContext) {
log.debug("Table definition tx start, params: {}", model);
// 1. 获取原有表定义
final CustomTableDefinitionModel existingTable = customTableDefinitionRepository
.queryOneInfoById(model.getId());
if (existingTable == null) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "表定义不存在");
}
final List<CustomFieldDefinitionModel> fieldsToAdd = new ArrayList<>();
final List<CustomFieldDefinitionModel> fieldsToUpdate = new ArrayList<>();
final List<CustomFieldDefinitionModel> fieldsToDelete = new ArrayList<>();
final List<String> dorisAlterSqls = new ArrayList<>();
final Map<String, CustomFieldDefinitionModel> existingFieldMapFinal;
final boolean hasDorisDataFinal;
// 检查Doris数据状态只需要检查一次
hasDorisDataFinal = customDorisTableDomainService.hasData(
existingTable.getDorisDatabase(),
existingTable.getDorisTable());
// 2. 处理字段变更 (MySQL),fieldList是前端给的字段定义
if (!CollectionUtils.isEmpty(fieldList)) {
// 获取现有字段列表并设为 final,model.getFieldList()是当前库里的字段定义
List<CustomFieldDefinitionModel> existingFields = model.getFieldList();
existingFieldMapFinal = existingFields.stream()
.collect(Collectors.toMap(CustomFieldDefinitionModel::getFieldName, field -> field));
Set<String> processedFields = new HashSet<>();
// 处理新增和更新的字段,新增的字段,如果 sortIndex字段没有值,则检查库里已有字段的sortIndex,在最大的 sortIndex基础上 +1
// ,是字段有序;一般是用前端的sortIndex字段排序,越小越靠前
for (CustomFieldDefinitionModel newField : fieldList) {
// 检查是否是系统默认字段
DefaultTableFieldEnum defaultField = DefaultTableFieldEnum.getEnumByFieldName(newField.getFieldName());
if (defaultField != null) {
// 如果是系统默认字段,使用默认配置覆盖用户的修改
overrideWithDefaultFieldDefinition(defaultField, newField);
// 如果字段已存在,则更新;否则添加
CustomFieldDefinitionModel existingField = existingFieldMapFinal.get(newField.getFieldName());
if (existingField != null) {
newField.setId(existingField.getId());
newField.setTableId(model.getId());
fieldsToUpdate.add(newField);
} else {
newField.setTableId(model.getId());
fieldsToAdd.add(newField);
}
processedFields.add(newField.getFieldName());
continue;
}
CustomFieldDefinitionModel existingField = existingFieldMapFinal.get(newField.getFieldName());
processedFields.add(newField.getFieldName());
if (existingField == null) {
// 新增字段
newField.setTableId(model.getId());
fieldsToAdd.add(newField);
} else {
// 更新字段
if (hasDorisDataFinal) {
// 如果Doris表有数据,则需要检查字段约束变更
validateFieldConstraintChanges(existingField, newField);
}
newField.setId(existingField.getId());
newField.setTableId(model.getId());
fieldsToUpdate.add(newField);
}
}
// 处理删除的字段(排除系统默认字段)
List<CustomFieldDefinitionModel> tempFieldsToDelete = existingFields.stream()
.filter(field -> !processedFields.contains(field.getFieldName()))
.filter(field -> DefaultTableFieldEnum.getEnumByFieldName(field.getFieldName()) == null) // 排除系统默认字段
.collect(Collectors.toList());
fieldsToDelete.addAll(tempFieldsToDelete);
// 如果Doris表有数据且有字段要删除抛出异常
if (hasDorisDataFinal && !fieldsToDelete.isEmpty()) {
log.error("Doris has data and columns to drop, abort, table: {}.{}, columns:{}", existingTable.getDorisDatabase(),
existingTable.getDorisTable(), JSON.toJSON(fieldsToDelete));
throw ComposeException.build(BizExceptionCodeEnum.composeCannotDropFieldWithData);
}
// 执行MySQL字段变更
if (!fieldsToAdd.isEmpty()) {
customFieldDefinitionRepository.batchAddInfo(fieldsToAdd, userContext);
}
if (!fieldsToUpdate.isEmpty()) {
customFieldDefinitionRepository.batchUpdateInfo(fieldsToUpdate, userContext);
}
if (!fieldsToDelete.isEmpty()) {
for (CustomFieldDefinitionModel field : fieldsToDelete) {
customFieldDefinitionRepository.deleteById(field.getId(), userContext);
}
}
} else {
// 如果传入的 fieldList 为空,则 existingFieldMap 需要初始化为空 Map
existingFieldMapFinal = Collections.emptyMap();
}
// 3. 更新MySQL表定义基本信息 (先更新MySQL确保事务一致性)
customTableDefinitionRepository.updateInfo(model, userContext);
// 4. 处理Doris表结构
if (!hasDorisDataFinal) {
// Doris无数据: 重建表
log.info("Doris empty, will recreate schema, table: {}.{}", existingTable.getDorisDatabase(),
existingTable.getDorisTable());
// 库表字段更新后,需要重新插最新的库表结构定义
var tableModel = customTableDefinitionRepository.queryOneInfoById(model.getId());
customDorisTableDomainService.rebuildTable(tableModel);
} else {
// Doris有数据: 生成并执行 ALTER TABLE 语句
var sqlList = tableDbWrapperUtil.generateDorisAlterSqls(existingTable, fieldsToAdd, fieldsToUpdate,
existingFieldMapFinal);
log.info("Doris has data, will ALTER, tableName:{}, sqlList:{}", existingTable.getTableName(),
JSON.toJSONString(sqlList));
dorisAlterSqls
.addAll(sqlList);
if (!dorisAlterSqls.isEmpty()) {
log.info("Doris has data, will ALTER, table: {}.{}", existingTable.getDorisDatabase(),
existingTable.getDorisTable());
customDorisTableDomainService.alterTable(existingTable.getDorisDatabase(),
existingTable.getDorisTable(), dorisAlterSqls);
}
}
log.debug("Table definition tx complete");
}
@Override
@DSTransactional(rollbackFor = Exception.class)
public CustomTableDefinitionModel deleteById(Long tableId, UserContext userContext) {
log.debug("Delete table definition start, tableId: {}", tableId);
// 1. 获取表定义
CustomTableDefinitionModel tableModel = customTableDefinitionRepository.queryOneInfoById(tableId);
if (tableModel == null) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "表定义不存在");
}
// 3. 删除表定义
customTableDefinitionRepository.deleteById(tableId);
// 删除关联的字段定义
customFieldDefinitionRepository.deleteByTableId(tableId);
log.debug("Delete table definition done");
return tableModel;
}
/**
* 验证字段约束变更
*/
private void validateFieldConstraintChanges(CustomFieldDefinitionModel existingField,
CustomFieldDefinitionModel newField) {
// 检查唯一性约束变更
if (!Objects.equals(existingField.getUniqueFlag(), newField.getUniqueFlag())) {
log.warn("Field [{}] uniqueness changed; abort if Doris has data", existingField.getFieldName());
throw ComposeException.build(BizExceptionCodeEnum.composeCannotChangeUniqueWithData, existingField.getFieldName());
}
// 检查非空约束变更,如果字段是从可空变更为不可空,则需要检查Doris表中是否存在数据,如果存在数据,则抛出异常
// if (!Objects.equals(existingField.getNullableFlag(),
// newField.getNullableFlag())) {
// if (DorisConfigContants.FIXED_FIELD_NULLABLE) {
// log.warn("Field [{}] nullability tightened; abort if Doris has data",
// existingField.getFieldName());
// throw ComposeException.build(BizExceptionCodeEnum.composeCannotChangeNullableWithData,
// existingField.getFieldName());
// }
// }
if (!Objects.equals(existingField.getFieldType(), newField.getFieldType())) {
log.warn("Field [{}] type changed; Doris ALTER may be risky");
}
}
/**
* 使用系统默认字段定义覆盖用户的修改
*/
private void overrideWithDefaultFieldDefinition(DefaultTableFieldEnum defaultField,
CustomFieldDefinitionModel field) {
TableFieldDefineVo defaultFieldVo = defaultField.toFieldDefineVo();
// 使用默认配置覆盖用户的修改
field.setFieldType(defaultFieldVo.getFieldType());
field.setNullableFlag(defaultFieldVo.getNullableFlag());
field.setUniqueFlag(defaultFieldVo.getUniqueFlag());
field.setEnabledFlag(defaultFieldVo.getEnabledFlag());
field.setDefaultValue(defaultFieldVo.getDefaultValue());
field.setFieldDescription(defaultField.getFieldDescription());
log.debug("System field [{}] reset to default config", defaultField.getFieldName());
}
@Override
public void exportTableDataToExcel(Long tableId, OutputStream outputStream, UserContext userContext) {
log.info("Export table to Excel start, tableId: {}", tableId);
if (tableId == null) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "表ID不能为空");
}
// 提前获取并检查 tableModel
CustomTableDefinitionModel tableModel = this.customTableDefinitionRepository.queryOneInfoById(tableId);
if (tableModel == null) {
log.error("Table definition not found for export, tableId: {}", tableId);
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "表定义不存在");
}
String database = tableModel.getDorisDatabase();
String table = tableModel.getDorisTable();
if (StrUtil.hasBlank(database, table)) {
log.error("Table definition has empty database or table name, tableId: {}, database: {}, table: {}", tableId, database, table);
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "表定义中数据库名或表名不能为空");
}
// 2. 获取字段定义列表 (仍然可能抛异常或返回空留在try里)
List<CustomFieldDefinitionModel> fields = tableModel.getFieldList();
if (CollUtil.isEmpty(fields)) {
log.warn("Field definitions missing or empty, tableId: {}", tableId);
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound);
}
try {
// 3. 查询表数据 (留在try里)
List<Map<String, Object>> tableData = customDorisTableDomainService.queryAllTableData(database, table, null,
"id ASC", null);
// 长数值字段处理对可能超过Integer范围的数值进行字符串转换避免Excel中的科学计数法问题
// 创建字段类型映射,用于判断哪些字段需要特殊处理
Map<String, Integer> fieldTypeMap = new HashMap<>();
for (CustomFieldDefinitionModel field : fields) {
fieldTypeMap.put(field.getFieldName(), field.getFieldType());
}
// 对数据进行预处理强制将所有数值类型字段转换为字符串确保Excel整列按字符串格式处理
for (Map<String, Object> rowData : tableData) {
if (rowData == null) {
continue; // 防御性检查:跳过空行
}
for (Map.Entry<String, Object> entry : rowData.entrySet()) {
if (entry == null) {
continue; // 防御性检查:跳过空条目
}
String fieldName = entry.getKey();
Object fieldValue = entry.getValue();
// 强制转换所有数值类型包括BigDecimal为字符串确保Excel按字符串格式处理
if (fieldValue instanceof Number) {
String stringValue = formatNumberForExcel(fieldValue);
rowData.put(fieldName, stringValue);
log.debug("Field [{}] value [{}] ({}) coerced to string [{}] for Excel",
fieldName, fieldValue, fieldValue.getClass().getSimpleName(), stringValue);
}
}
}
// 4. 转换数据格式为Excel写入格式 (留在try里)
List<List<Object>> excelData = new ArrayList<>();
List<Object> titleRow = new ArrayList<>();
for (CustomFieldDefinitionModel field : fields) {
if (field.getEnabledFlag() != null && field.getEnabledFlag() == 1) {
// 这里使用英文字段和中文名称,格式: 中文名称(英文字段名)
String header = String.format("%s(%s)",
StringUtils.hasText(field.getFieldDescription()) ? field.getFieldDescription()
: field.getFieldName(),
field.getFieldName());
titleRow.add(header);
}
}
excelData.add(titleRow);
for (Map<String, Object> rowData : tableData) {
List<Object> excelRow = new ArrayList<>();
for (CustomFieldDefinitionModel field : fields) {
if (field.getEnabledFlag() != null && field.getEnabledFlag() == 1) {
String fieldName = field.getFieldName();
Object fieldValue = rowData.get(fieldName);
if (fieldValue instanceof Date date) {
fieldValue = DateUtil.format(date, "yyyy-MM-dd HH:mm:ss");
} else if (fieldValue instanceof LocalDateTime localDateTime) {
fieldValue = localDateTime.format(DATE_TIME_FORMATTER);
}
excelRow.add(fieldValue);
}
}
excelData.add(excelRow);
}
// 5. 使用FastExcel写入数据 (留在try里)
FastExcel.write(outputStream)
.sheet(table)
.doWrite(excelData);
outputStream.flush();
log.info("Export to Excel done, tableId: {}, database: {}, table: {}, rows: {}",
tableId, database, table, excelData.size());
} catch (Exception e) {
// 记录完整的异常堆栈信息
log.error("Export to Excel error, tableId: {}, database: {}, table: {}",
tableId, database, table, e); // Can use variables defined outside try
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "导出Excel时发生异常: " + e.getMessage());
}
}
@Override
public CustomTableDefinitionModel queryOneTableInfoById(Long tableId) {
log.debug("Query table definition, tableId: {}", tableId);
return this.customTableDefinitionRepository.queryOneInfoById(tableId);
}
@Override
public List<CustomTableDefinitionModel> queryAllTableDefineList(DorisToolTableDefineRequest request) {
log.debug("Query table definition list, request: {}", request);
if (request == null) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "请求参数不能为空");
}
// 必须传入租户ID
if (request.getTenantId() == null) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "租户ID不能为空");
}
// 构建查询条件
CustomTableDefinitionModel queryModel = new CustomTableDefinitionModel();
queryModel.setTenantId(request.getTenantId());
// 可选条件空间ID
if (request.getSpaceId() != null) {
queryModel.setSpaceId(request.getSpaceId());
}
// 可选条件用户ID如果需要根据用户ID过滤的话
// 注意这里需要根据实际业务需求来决定如何使用userId进行过滤
// 如果用户ID是用来过滤用户有权限访问的表可能需要额外的权限表查询
// 调用 repository 层查询
List<CustomTableDefinitionModel> tableList = customTableDefinitionRepository.queryListByCondition(queryModel);
// 如果列表为空,返回空列表而不是 null
if (CollectionUtils.isEmpty(tableList)) {
return Collections.emptyList();
}
log.debug("Found {} table definitions", tableList.size());
return tableList;
}
@Override
public String getCreateTableDdl(Long tableId, CustomTableDefinitionModel tableDefine) {
var database = tableDefine.getDorisDatabase();
var table = tableDefine.getDorisTable();
var ddl = this.customDorisTableRepository.getCreateTableDdl(database, table);
return ddl;
}
@Override
public SuperPage<CustomTableDefinitionModel> queryPageTableDefine(QueryDorisTableDefinePageRequest request) {
// 构建查询条件
CustomTableDefinitionModel queryModel = new CustomTableDefinitionModel();
queryModel.setSpaceId(request.getSpaceId());
// 调用 repository 层查询
List<CustomTableDefinitionModel> tableList = customTableDefinitionRepository.queryListByCondition(queryModel);
// 将列表转换为分页对象
SuperPage<CustomTableDefinitionModel> page = new SuperPage<>();
page.setRecords(tableList);
page.setTotal(tableList.size());
page.setSize(request.getPageSize());
page.setCurrent(request.getPageNo());
return page;
}
@Override
public void updateTableName(CustomTableDefinitionModel model, UserContext userContext) {
this.customTableDefinitionRepository.updateTableName(model, userContext);
}
@Override
public Long copyTableDefinition(Long tableId, UserContext userContext) {
var tableModel = this.customTableDefinitionRepository.queryOneInfoById(tableId);
if (tableModel == null) {
log.error("Table definition not found, tableId: {}", tableId);
throw ComposeException.build(BizExceptionCodeEnum.composeTableDefinitionNotFound);
}
var newTableId = this.customTableDefinitionRepository.copyTableDefinition(tableId, userContext);
return newTableId;
}
@Override
public void updateTableLastModified(Long tableId, UserContext userContext) {
this.customTableDefinitionRepository.updateTableLastModified(tableId, userContext);
}
@Override
public FieldCreationResult handleNewFieldsFromExcel(CustomTableDefinitionModel tableModel,
List<String> newFieldsToCreate,
UserContext userContext) {
List<CustomFieldDefinitionModel> existingFields = new ArrayList<>(tableModel.getFieldList());
Map<String, String> newFieldMapping = new java.util.HashMap<>();
List<CustomFieldDefinitionModel> newFieldModels = new ArrayList<>();
// 找出当前自定义字段名中数字后缀的最大值
int maxCustomIndex = existingFields.stream()
.map(CustomFieldDefinitionModel::getFieldName)
.filter(name -> name.matches("c_\\d+"))
.map(name -> name.substring(2))
.mapToInt(Integer::parseInt)
.max()
.orElse(0);
// 找出当前最大的排序索引
int maxSortIndex = existingFields.stream()
.mapToInt(CustomFieldDefinitionModel::getSortIndex)
.max()
.orElse(-1);
// 根据excel的表头调用大模型生成英文字段名
Map<String, String> englishFieldNameMap = this.getEnglishFieldNameFromExcelHeader(newFieldsToCreate);
// 获取现有字段的英文名集合,用于冲突检测
Set<String> existingFieldNames = existingFields.stream()
.map(CustomFieldDefinitionModel::getFieldName)
.collect(Collectors.toSet());
// 用于存储本次循环中已经分配的英文名,防止内部冲突(例如多个中文翻译成同一个英文)
Set<String> newlyAssignedEnglishNames = new HashSet<>();
for (String chineseHeader : newFieldsToCreate) {
String proposedEnglishName = englishFieldNameMap.get(chineseHeader);
// 清理字段名中的空白字符
proposedEnglishName = ExcelUtils.cleanWhitespace(proposedEnglishName);
String finalFieldName;
// 检查翻译的英文名是否可用 (不与现有字段冲突,也不与本次已分配的字段冲突)
if (StringUtils.hasText(proposedEnglishName) &&
!existingFieldNames.contains(proposedEnglishName) &&
!newlyAssignedEnglishNames.contains(proposedEnglishName)) {
finalFieldName = proposedEnglishName;
newlyAssignedEnglishNames.add(finalFieldName); // 记录已分配的英文名
} else {
// 如果冲突或翻译为空,则使用 c_X 格式
maxCustomIndex++; // maxCustomIndex 是之前计算得到的,这里继续递增
finalFieldName = "c_" + maxCustomIndex;
// 确保 c_X 格式的名称也不会与现有字段名或本次已分配的字段名冲突
while (existingFieldNames.contains(finalFieldName) || newlyAssignedEnglishNames.contains(finalFieldName)) {
maxCustomIndex++;
finalFieldName = "c_" + maxCustomIndex;
}
newlyAssignedEnglishNames.add(finalFieldName); // 记录已分配的 c_X 格式的英文名
}
CustomFieldDefinitionModel newField = CustomFieldDefinitionModel.createFromExcelHeader(
tableModel,
chineseHeader,
finalFieldName, // 使用最终确定的字段名
++maxSortIndex,
userContext);
newFieldModels.add(newField);
// 更新 newFieldMapping确保它反映的是最终使用的字段名
newFieldMapping.put(chineseHeader, finalFieldName);
}
// 合并新旧字段
List<CustomFieldDefinitionModel> allFields = new ArrayList<>(existingFields);
allFields.addAll(newFieldModels);
// 调用服务更新表定义
this.self.updateInfo(tableModel, allFields, userContext);
// 获取最新的表定义
CustomTableDefinitionModel updatedTableModel = this.self.queryOneTableInfoById(tableModel.getId());
return new FieldCreationResult(updatedTableModel, newFieldMapping);
}
/**
* 通过excel表头调用大模型生成英文字段名
*
* @param newFieldsToCreate
* @return
*/
private Map<String, String> getEnglishFieldNameFromExcelHeader(List<String> newFieldsToCreate) {
// 构建调用大模型的请求
String sysPrompt = "你是一个专业的翻译引擎,专门负责将中文列表中的表头字段翻译成符合编程规范的英文变量名。\n" +
"请遵循以下规则:\n" +
"1. **准确翻译**:确保英文翻译准确反映中文含义。\n" +
"2. **小写蛇形命名法 (snake_case)**:所有字母小写,单词间用下划线 `_` 连接。\n" +
"3. **简洁性**:翻译结果应尽可能简洁明了。\n" +
"4. **JSON输出**请将结果组织成一个JSON对象其中每个键是原始的中文表头对应的值是翻译后的英文变量名。\n\n" +
"例如,如果输入是:`[\"用户名称\", \"登录密码\"]`\n" +
"期望的输出 (JSON字符串) 应该是:\n" +
"`{\\\"用户名称\\\": \\\"user_name\\\", \\\"登录密码\\\": \\\"login_password\\\"}`";
// 使用 fastjson2
String jsonData = com.alibaba.fastjson2.JSON.toJSONString(newFieldsToCreate);
String userPrompt = "请处理以下中文表头列表并严格按照系统提示中定义的JSON格式返回结果\n" + jsonData;
// 调用大模型
Map<String, String> result = modelApplicationService.call(sysPrompt, userPrompt,
new org.springframework.core.ParameterizedTypeReference<Map<String, String>>() {
});
return result;
}
/**
* 判断数值是否需要转换为字符串以避免Excel中的显示问题
*
* @param fieldValue 字段值
* @param fieldType 字段类型
* @return 是否需要转换为字符串
*/
private boolean shouldConvertToString(Object fieldValue, Integer fieldType) {
// 基础检查字段值必须是非null的数值类型
if (fieldValue == null || !(fieldValue instanceof Number)) {
return false;
}
// 字段类型检查如果字段类型为null则不进行转换
if (fieldType == null) {
log.debug("Field type null, skip numeric conversion");
return false;
}
// 配置检查:如果未启用智能转换,则不进行处理
if (!DorisConfigContants.ENABLE_SMART_NUMBER_CONVERSION) {
return false;
}
Number number = (Number) fieldValue;
try {
// 主键类型根据配置决定是否转换为字符串
if (TableFieldTypeEnum.PRIMARY_KEY.getCode().equals(fieldType)) {
return DorisConfigContants.FORCE_CONVERT_PRIMARY_KEY;
}
// INTEGER类型超过Integer范围时转换为字符串
if (TableFieldTypeEnum.INTEGER.getCode().equals(fieldType)) {
long longValue = number.longValue();
return longValue > Integer.MAX_VALUE || longValue < Integer.MIN_VALUE;
}
// NUMBER类型处理大数值和高精度数值
if (TableFieldTypeEnum.NUMBER.getCode().equals(fieldType)) {
// 方案1检查数值字符串长度是否超过Excel安全显示范围
String numberStr = number.toString();
if (numberStr != null) {
// 移除小数点、负号和科学计数法标记来计算纯数字位数
String digitsOnly = numberStr.replaceAll("[-.eE]", "");
if (digitsOnly.length() > DorisConfigContants.EXCEL_MAX_PRECISION_DIGITS) {
return true;
}
}
// 方案2检查绝对值是否超过Excel精度范围
double doubleValue = number.doubleValue();
if (Math.abs(doubleValue) >= DorisConfigContants.EXCEL_MAX_SAFE_NUMBER) {
return true;
}
// 方案3特殊处理BigDecimal类型的高精度数值
if (fieldValue instanceof BigDecimal) {
BigDecimal bd = (BigDecimal) fieldValue;
// 如果整数部分超过Excel精度限制则转换为字符串
if (bd.precision() - bd.scale() > DorisConfigContants.EXCEL_MAX_PRECISION_DIGITS) {
return true;
}
}
}
} catch (Exception e) {
// 如果在处理过程中发生任何异常,记录错误但不影响整体流程
log.error("Numeric conversion check error, fieldValue: {}, fieldType: {}", fieldValue, fieldType, e);
return false;
}
return false;
}
/**
* 判断字段类型是否为数值类型
*
* @param fieldType 字段类型
* @return 是否为数值类型
*/
private boolean isNumericFieldType(Integer fieldType) {
if (fieldType == null) {
return false;
}
return TableFieldTypeEnum.PRIMARY_KEY.getCode().equals(fieldType) ||
TableFieldTypeEnum.INTEGER.getCode().equals(fieldType) ||
TableFieldTypeEnum.NUMBER.getCode().equals(fieldType);
}
/**
* 格式化数值为适合Excel导出的字符串格式
* 确保数值精度不丢失且Excel不会按科学计数法显示
*
* @param fieldValue 字段值
* @return 格式化后的字符串
*/
private String formatNumberForExcel(Object fieldValue) {
if (fieldValue == null) {
return "";
}
try {
log.debug("Format numeric: [{}], type: [{}]", fieldValue, fieldValue.getClass().getSimpleName());
// 优先处理BigDecimal类型使用toPlainString()避免科学计数法
if (fieldValue instanceof BigDecimal) {
String result = ((BigDecimal) fieldValue).toPlainString();
log.debug("BigDecimal [{}] -> string [{}]", fieldValue, result);
return result;
}
// 对于Double类型转换为BigDecimal以保持精度
if (fieldValue instanceof Double) {
Double doubleValue = (Double) fieldValue;
BigDecimal bd = BigDecimal.valueOf(doubleValue);
String result = bd.toPlainString();
log.debug("Double [{}] -> string [{}]", fieldValue, result);
return result;
}
// 对于Float类型转换为BigDecimal以保持精度
if (fieldValue instanceof Float) {
Float floatValue = (Float) fieldValue;
BigDecimal bd = BigDecimal.valueOf(floatValue.doubleValue());
String result = bd.toPlainString();
log.debug("Float [{}] -> string [{}]", fieldValue, result);
return result;
}
// 对于Long、Integer等整数类型直接转换
if (fieldValue instanceof Long || fieldValue instanceof Integer ||
fieldValue instanceof Short || fieldValue instanceof Byte) {
String result = fieldValue.toString();
log.debug("Integer [{}] -> string [{}]", fieldValue, result);
return result;
}
// 其他数值类型统一使用BigDecimal处理
if (fieldValue instanceof Number) {
Number number = (Number) fieldValue;
BigDecimal bd = new BigDecimal(number.toString());
String result = bd.toPlainString();
log.debug("Other numeric [{}] -> string [{}]", fieldValue, result);
return result;
}
// 非数值类型直接toString
String result = fieldValue.toString();
log.debug("Non-numeric [{}] -> string [{}]", fieldValue, result);
return result;
} catch (Exception e) {
log.error("Format numeric [{}] to Excel string error", fieldValue, e);
// 异常时使用默认转换
return String.valueOf(fieldValue);
}
}
@Override
public List<CustomTableDefinitionModel> queryTableDefineBySpaceId(Long spaceId) {
return this.customTableDefinitionRepository.queryListBySpaceId(spaceId);
}
@Override
public Long countTotalTables() {
return this.customTableDefinitionRepository.countTotalTables(null);
}
}

View File

@@ -0,0 +1,88 @@
package com.xspaceagi.compose.domain.util;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.util.Date;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.NoArgsConstructor;
import org.springframework.util.StringUtils;
import lombok.extern.slf4j.Slf4j;
/**
* sql 构建工具
*/
@NoArgsConstructor
@Slf4j
public class BuildSqlUtil {
/**
* 构建插入值字符串
*/
public static String buildInsertValues(Map<String, Object> rowData) {
return rowData.entrySet().stream()
.map(entry -> {
String key = entry.getKey();
Object value = entry.getValue();
return "`" + escapeSqlString(key) + "` = " + formatInsertValue(value);
})
.collect(Collectors.joining(", "));
}
/**
* 格式化插入值
*/
private static String formatInsertValue(Object value) {
if (value == null) {
return "";
}
if (value instanceof String) {
return "'" + escapeSqlString((String) value) + "'";
}
if (value instanceof Number) {
return value.toString();
}
if (value instanceof Boolean) {
return ((Boolean) value) ? "1" : "0";
}
if (value instanceof Date) {
return "'" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(value) + "'";
}
if (value instanceof LocalDateTime) {
return "'" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(value) + "'";
}
return value.toString();
}
/**
* 转义 SQL 字符串中的特殊字符 (简单实现)
* 主要处理单引号和反斜杠
*
* @param value 需要转义的字符串
* @return 转义后的字符串
*/
public static String escapeSqlString(String value) {
if (!StringUtils.hasText(value)) {
return "";
}
// 替换反斜杠为双反斜杠,替换单引号为反斜杠+单引号
// Java String literal: \ becomes \\, ' becomes \'
return value.replace("\\", "\\\\").replace("'", "\\'");
}
/**
* 构建更新值字符串
*/
public static String buildUpdateValues(Map<String, Object> rowData) {
return rowData.entrySet().stream()
.map(entry -> {
String key = entry.getKey();
Object value = entry.getValue();
return "`" + escapeSqlString(key) + "` = " + formatInsertValue(value);
})
.collect(Collectors.joining(", "));
}
}

View File

@@ -0,0 +1,632 @@
package com.xspaceagi.compose.domain.util;
import com.alibaba.fastjson2.JSON;
import com.xspaceagi.compose.domain.config.DorisProperties;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.domain.util.dml.DeleteTableDmlUtil;
import com.xspaceagi.compose.domain.util.dml.InsertTableDMLUtil;
import com.xspaceagi.compose.domain.util.dml.UpdateTableDMLUtil;
import com.xspaceagi.compose.sdk.enums.DefaultTableFieldEnum;
import com.xspaceagi.compose.sdk.enums.TableFieldTypeEnum;
import com.xspaceagi.compose.spec.constants.DorisConfigContants;
import com.xspaceagi.system.spec.exception.BizExceptionCodeEnum;
import com.xspaceagi.system.spec.exception.ComposeException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* Doris DDL语句生成工具类
*/
@Slf4j
@Component
public class DorisTableDdlUtil {
/**
* 静态变量,用于存储 DorisProperties
*/
private static DorisProperties dorisProperties;
@Autowired
public void setDorisProperties(DorisProperties dorisProperties) {
DorisTableDdlUtil.dorisProperties = dorisProperties;
log.info("[DorisProperties] Doris config at startup: {}", JSON.toJSONString(dorisProperties));
}
/**
* 构建创建表的SQL语句
*
* @param tableModel 表定义模型
* @param fields 字段定义列表
* @return CREATE TABLE SQL 语句
*/
public static String buildCreateTableSql(CustomTableDefinitionModel tableModel,
List<CustomFieldDefinitionModel> fields) {
if (tableModel == null || !StringUtils.hasText(tableModel.getDorisDatabase())
|| !StringUtils.hasText(tableModel.getDorisTable())) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "表定义、数据库名或表名不能为空");
}
if (CollectionUtils.isEmpty(fields)) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "创建表时至少需要一个字段");
}
// 固定 id 作为唯一键
List<String> uniqueKeyFields = new java.util.ArrayList<>();
uniqueKeyFields.add("id");
int bucketNum = getBucketNum();
StringBuilder sql = new StringBuilder();
// 1. CREATE TABLE 语句头
sql.append("CREATE TABLE IF NOT EXISTS `")
.append(tableModel.getDorisDatabase()).append("`.`")
.append(tableModel.getDorisTable()).append("` (\n");
// 2. 添加 id 主键字段
sql.append(" `id` BIGINT NOT NULL COMMENT '主键ID',\n");
// 3. 字段定义
String columnDefinitions = fields.stream()
// 过滤掉id字段
.filter(f -> !f.getFieldName().equals("id"))
.map(DorisTableDdlUtil::buildColumnDefinitionSql)
.collect(Collectors.joining(",\n"));
sql.append(columnDefinitions).append("\n");
// 4. 表属性
sql.append(") \n")
.append("ENGINE=OLAP\n")
.append("UNIQUE KEY(`id`)\n")
.append("COMMENT '").append(BuildSqlUtil.escapeSqlString(tableModel.getTableDescription()))
.append("'\n")
.append("DISTRIBUTED BY HASH(`id`) BUCKETS ").append(bucketNum).append("\n")
.append("PROPERTIES (\n")
.append(" \"replication_num\" = \"").append(getReplicationNum()).append("\",")
.append("\n \"enable_unique_key_merge_on_write\" = \"true\"\n")
.append(")");
log.debug("Built CREATE TABLE SQL for {}.{}: \n{}", tableModel.getDorisDatabase(), tableModel.getDorisTable(),
sql.toString());
return sql.toString();
}
/**
* 获取重复键字段名
* 如果配置文件中指定了字段名,则使用配置的字段名
* 否则使用第一个字段作为重复键
*/
private static String getDuplicateKey(List<CustomFieldDefinitionModel> fields) {
if (dorisProperties != null
&& StringUtils.hasText(dorisProperties.getDuplicateKey())) {
return dorisProperties.getDuplicateKey();
}
return fields.get(0).getFieldName(); // 默认使用第一个字段
}
/**
* 获取分布键字段名
* 如果配置文件中指定了字段名,则使用配置的字段名
* 否则使用第一个字段作为分布键
*/
private static String getDistributedKey(List<CustomFieldDefinitionModel> fields) {
if (dorisProperties != null
&& StringUtils.hasText(dorisProperties.getDistributedKey())) {
return dorisProperties.getDistributedKey();
}
return fields.get(0).getFieldName(); // 默认使用第一个字段
}
/**
* 获取分桶数
*/
private static int getBucketNum() {
if (dorisProperties != null
&& dorisProperties.getBucketNum() != null) {
return dorisProperties.getBucketNum();
}
return 10; // 默认值
}
/**
* 获取副本数
* 如果配置类未初始化则返回默认值3
*/
private static int getReplicationNum() {
if (dorisProperties != null
&& dorisProperties.getReplicationNum() != null) {
return dorisProperties.getReplicationNum();
}
return 3; // 默认值
}
/**
* 构建单个字段的 SQL 定义部分
*
* @param field 字段定义模型
* @return 单个字段的 SQL 定义字符串
*/
private static String buildColumnDefinitionSql(CustomFieldDefinitionModel field) {
if (field == null || !StringUtils.hasText(field.getFieldName())) {
log.warn("Skipping invalid field definition");
return ""; // Or throw exception, depending on business logic
}
StringBuilder fieldSql = new StringBuilder();
// Field name, quoted
fieldSql.append(" `").append(field.getFieldName()).append("` ");
// 特殊处理 modified 字段
if ("modified".equalsIgnoreCase(field.getFieldName())) {
fieldSql.append("DATETIME DEFAULT CURRENT_TIMESTAMP");
} else {
// Field type
fieldSql.append(convertToDorisTypeDefinition(field.getFieldType()));
// Mark if NOT NULL
boolean isNotNull = field.getNullableFlag() != null && field.getNullableFlag() == -1;
// 读取控制字段: DorisConfigContants.FIXED_FIELD_NULLABLE ,建表的时候,不限制字段
if (DorisConfigContants.FIXED_FIELD_NULLABLE) {
isNotNull = false;
}
// Nullability constraint
if (isNotNull) {
fieldSql.append(" NOT NULL");
} else {
fieldSql.append(" NULL"); // Explicitly add NULL
}
// Default value handling
if (field.getDefaultValue() != null) { // Check if a default value was specified in the model
String formattedDefault = formatDefaultValue(field.getDefaultValue(), field.getFieldType());
// Only add DEFAULT clause if the default value is valid (formattedDefault !=
// null)
// AND (the column is nullable OR the default value is not "NULL")
if (formattedDefault != null && (!isNotNull || !"NULL".equals(formattedDefault))) {
fieldSql.append(" DEFAULT ").append(formattedDefault);
} else if (formattedDefault != null && isNotNull && "NULL".equals(formattedDefault)) {
// Log a warning if the column is NOT NULL but the default resolves to NULL
log.warn("Field '{}' is NOT NULL but default resolves to NULL; no DEFAULT clause will be added.", field.getFieldName());
}
}
}
// Field comment
if (StringUtils.hasText(field.getFieldDescription())) {
fieldSql.append(" COMMENT '").append(BuildSqlUtil.escapeSqlString(field.getFieldDescription())).append("'");
}
return fieldSql.toString();
}
/**
* 将内部字段类型代码转换为Doris数据类型定义字符串
*
* @param fieldTypeCode 内部字段类型代码 (来自 TableFieldTypeEnum 的 code)
* @return Doris 数据类型定义字符串 (例如 "VARCHAR(255)", "INT")
*/
private static String convertToDorisTypeDefinition(Integer fieldTypeCode) {
if (fieldTypeCode == null) {
log.warn("Field type code is null, returning default definition: {}", TableFieldTypeEnum.STRING.getDorisDefinition());
return TableFieldTypeEnum.STRING.getDorisDefinition();
}
for (TableFieldTypeEnum typeEnum : TableFieldTypeEnum.values()) {
if (typeEnum.getCode().equals(fieldTypeCode)) {
return typeEnum.getDorisDefinition();
}
}
log.warn("No TableFieldTypeEnum for code {}, default def: {}",
fieldTypeCode, TableFieldTypeEnum.STRING.getDorisDefinition());
return TableFieldTypeEnum.STRING.getDorisDefinition();
}
/**
* 格式化 SQL 的默认值
*
* @param defaultValue 默认值字符串
* @param fieldTypeCode 字段类型代码
* @return 格式化后的默认值字符串 (例如 'abc', 123, 'NULL')
*/
private static String formatDefaultValue(String defaultValue, Integer fieldTypeCode) {
// 空或 NULL 字符串直接返回 SQL NULL
if (!StringUtils.hasText(defaultValue) || "NULL".equalsIgnoreCase(defaultValue.trim())) {
return "NULL";
}
TableFieldTypeEnum typeEnum = TableFieldTypeEnum.STRING; // 默认按字符串处理
if (fieldTypeCode != null) {
for (TableFieldTypeEnum t : TableFieldTypeEnum.values()) {
if (t.getCode().equals(fieldTypeCode)) {
typeEnum = t;
break;
}
}
}
if (typeEnum == TableFieldTypeEnum.STRING && fieldTypeCode != null
&& !TableFieldTypeEnum.STRING.getCode().equals(fieldTypeCode)) {
log.warn("No enum for field type code {}, default as STRING", fieldTypeCode);
}
switch (typeEnum) {
case INTEGER:
case NUMBER:
// 只允许数字,非法则转字符串
try {
Double.parseDouble(defaultValue);
return defaultValue;
} catch (NumberFormatException e) {
log.warn("Field is numeric but default '{}' is invalid, treating as string", defaultValue);
return "'" + BuildSqlUtil.escapeSqlString(defaultValue) + "'";
}
case BOOLEAN:
if ("true".equalsIgnoreCase(defaultValue) || "1".equals(defaultValue))
return "1";
if ("false".equalsIgnoreCase(defaultValue) || "0".equals(defaultValue))
return "0";
log.warn("Field is boolean but default '{}' is invalid, using '0'", defaultValue);
return "0";
case DATE:
if ("CURRENT_TIMESTAMP".equalsIgnoreCase(defaultValue.trim())) {
return "CURRENT_TIMESTAMP";
}
log.warn("DATETIME field: default '{}' is not CURRENT_TIMESTAMP; no DEFAULT clause.", defaultValue);
return null;
case MEDIUMTEXT:
case STRING:
// 长文本和字符串类型,始终加引号
return "'" + BuildSqlUtil.escapeSqlString(defaultValue) + "'";
default:
// 其它类型默认加引号
return "'" + BuildSqlUtil.escapeSqlString(defaultValue) + "'";
}
}
// --- ALTER TABLE 相关方法 ---
/**
* 比较新旧字段列表,生成 ALTER TABLE 语句列表。
* 目前支持:
* 1. ADD COLUMN: 添加新字段。
* 2. MODIFY COLUMN COMMENT: 修改现有字段的注释。
* 注意:不支持删除字段、修改字段类型、修改可空性、修改默认值等。
*
* @param database 数据库名
* @param table 表名
* @param oldFields 旧的字段定义列表 (当前数据库中的结构)
* @param newFields 新的字段定义列表 (用户期望的结构)
* @return List<String> 包含 ALTER TABLE SQL 语句的列表,如果没有变更则为空列表。
*/
public static List<String> buildAlterTableSqls(String database, String table,
List<CustomFieldDefinitionModel> oldFields,
List<CustomFieldDefinitionModel> newFields) {
List<String> alterStatements = new ArrayList<>();
if (!StringUtils.hasText(database) || !StringUtils.hasText(table)) {
log.error("Database name or table name cannot be empty");
return alterStatements; // 返回空列表
}
if (newFields == null) {
newFields = new ArrayList<>(); // 避免空指针
}
if (oldFields == null) {
oldFields = new ArrayList<>(); // 避免空指针
}
// 使用字段名作为 Key 创建 Map方便查找
Map<String, CustomFieldDefinitionModel> oldFieldMap = oldFields.stream()
.filter(f -> f != null && StringUtils.hasText(f.getFieldName()))
.collect(Collectors.toMap(CustomFieldDefinitionModel::getFieldName, Function.identity(),
(f1, f2) -> f1)); // 重复时取第一个
Map<String, CustomFieldDefinitionModel> newFieldMap = newFields.stream()
.filter(f -> f != null && StringUtils.hasText(f.getFieldName()))
.collect(Collectors.toMap(CustomFieldDefinitionModel::getFieldName, Function.identity(),
(f1, f2) -> f1)); // 重复时取第一个
// --- 检查需要添加的字段 ---
for (Map.Entry<String, CustomFieldDefinitionModel> entry : newFieldMap.entrySet()) {
String fieldName = entry.getKey();
CustomFieldDefinitionModel newField = entry.getValue();
if (!oldFieldMap.containsKey(fieldName)) {
// 新增字段
String addColumnSql = buildAddColumnSql(newField);
if (StringUtils.hasText(addColumnSql)) {
alterStatements.add(String.format("ALTER TABLE `%s`.`%s` ADD COLUMN %s",
database, table, addColumnSql));
}
}
}
// --- 检查需要修改注释的字段 ---
for (Map.Entry<String, CustomFieldDefinitionModel> entry : newFieldMap.entrySet()) {
String fieldName = entry.getKey();
CustomFieldDefinitionModel newField = entry.getValue();
if (oldFieldMap.containsKey(fieldName)) {
CustomFieldDefinitionModel oldField = oldFieldMap.get(fieldName);
// 比较注释是否变化 (注意 null 处理)
String oldComment = oldField.getFieldDescription() == null ? "" : oldField.getFieldDescription();
String newComment = newField.getFieldDescription() == null ? "" : newField.getFieldDescription();
if (!Objects.equals(oldComment, newComment)) {
// 修改注释 (需要字段的完整定义,除了注释部分是新的)
// 注意: Doris 的 MODIFY COLUMN 语法通常需要提供完整的列定义
String modifyCommentSql = buildModifyColumnCommentSql(newField);
if (StringUtils.hasText(modifyCommentSql)) {
alterStatements.add(String.format("ALTER TABLE `%s`.`%s` MODIFY COLUMN %s",
database, table, modifyCommentSql));
}
}
// TODO: 在此添加对类型、可空性、默认值变化的比较和处理逻辑
}
}
// --- 检查需要删除的字段 ---
for (Map.Entry<String, CustomFieldDefinitionModel> entry : oldFieldMap.entrySet()) {
String fieldName = entry.getKey();
if (!newFieldMap.containsKey(fieldName)) {
// 旧字段在新字段列表中不存在,需要生成 DROP COLUMN 语句
// 注意:调用者需要根据业务规则(例如表是否有数据)来决定是否执行此语句
log.warn("Column marked for drop: {}; ensure no data before DROP!", fieldName);
alterStatements.add(String.format("ALTER TABLE `%s`.`%s` DROP COLUMN `%s`",
database, table, fieldName));
}
}
if (!alterStatements.isEmpty()) {
log.info("Generated {} ALTER statements for {}.{}: {}", database, table, alterStatements.size(), alterStatements);
}
return alterStatements;
}
/**
* 构建 ADD COLUMN 子句 (不包含 ALTER TABLE ... ADD COLUMN)
* 例如: `column_name` VARCHAR(255) NULL DEFAULT 'abc' COMMENT 'comment'
*/
private static String buildAddColumnSql(CustomFieldDefinitionModel field) {
if (field == null || !StringUtils.hasText(field.getFieldName())) {
log.warn("Cannot ADD COLUMN for invalid field");
return null;
}
StringBuilder sql = new StringBuilder();
sql.append("`").append(field.getFieldName()).append("` ");
sql.append(convertToDorisTypeDefinition(field.getFieldType()));
if (field.getNullableFlag() != null && field.getNullableFlag() == -1) {
sql.append(" NOT NULL");
} else {
sql.append(" NULL"); // 显式添加 NULL
}
if (field.getDefaultValue() != null) {
sql.append(" DEFAULT ").append(formatDefaultValue(field.getDefaultValue(), field.getFieldType()));
}
if (StringUtils.hasText(field.getFieldDescription())) {
sql.append(" COMMENT '").append(BuildSqlUtil.escapeSqlString(field.getFieldDescription())).append("'");
}
return sql.toString();
}
/**
* 构建 MODIFY COLUMN 子句,主要用于修改注释 (不包含 ALTER TABLE ... MODIFY COLUMN)
* 注意Doris 通常要求提供完整的列定义。
* 例如: `column_name` VARCHAR(255) NULL DEFAULT 'abc' COMMENT 'new_comment'
*/
private static String buildModifyColumnCommentSql(CustomFieldDefinitionModel field) {
// 实现与 buildAddColumnSql 几乎相同,因为需要提供完整的列定义
return buildAddColumnSql(field);
}
/**
* 构建创建数据库的SQL语句
*/
public static String buildCreateDatabaseSql(String database) {
return String.format("CREATE DATABASE IF NOT EXISTS `%s`", BuildSqlUtil.escapeSqlString(database));
}
/**
* 构建删除数据库的SQL语句
*/
public static String buildDropDatabaseSql(String database) {
return String.format("DROP DATABASE IF EXISTS `%s`", BuildSqlUtil.escapeSqlString(database));
}
/**
* 构建删除表的SQL语句
*/
public static String buildDropTableSql(String database, String table) {
return String.format("DROP TABLE IF EXISTS `%s`.`%s`",
BuildSqlUtil.escapeSqlString(database), BuildSqlUtil.escapeSqlString(table));
}
/**
* 获取表的建表DDL语句
*
* @param database 数据库名
* @param table 表名
* @return 建表DDL语句
*/
public static String getCreateTableDdl(String database, String table) {
if (!StringUtils.hasText(database) || !StringUtils.hasText(table)) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "数据库名或表名不能为空");
}
return "SHOW CREATE TABLE `" + database + "`.`" + table + "`";
}
/**
* 获取检查表是否存在的SQL语句
*
* @param database 数据库名
* @param table 表名
* @return 检查表是否存在的SQL语句
*/
public static String getTableExistsSql(String database, String table) {
if (!StringUtils.hasText(database) || !StringUtils.hasText(table)) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "数据库名或表名不能为空");
}
return "SELECT count(*) FROM information_schema.tables WHERE table_schema = '"
+ BuildSqlUtil.escapeSqlString(database) +
"' AND table_name = '" + BuildSqlUtil.escapeSqlString(table) + "' LIMIT 1";
}
/**
* 获取查询表总记录数的SQL语句
*
* @param database 数据库名
* @param table 表名
* @return 查询表总记录数的SQL语句
*/
public static String getTableCountSql(String database, String table) {
if (!StringUtils.hasText(database) || !StringUtils.hasText(table)) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "数据库名或表名不能为空");
}
return "SELECT COUNT(*) FROM `" + BuildSqlUtil.escapeSqlString(database) + "`.`"
+ BuildSqlUtil.escapeSqlString(table) + "`";
}
/**
* 构建插入数据的SQL语句 (使用 INSERT INTO ... SET ... 语法)
*/
public static String buildInsertSql(CustomTableDefinitionModel tableModel, Map<String, Object> rowData) {
// doris生成插入语句,需要主动设置id主键,没有自动生成,使用雪花算法生成id
return InsertTableDMLUtil.buildDorisInsertSql(tableModel, rowData);
}
/**
* 构建更新数据的SQL语句,根据id来修改行数
*/
public static String buildUpdateSql(CustomTableDefinitionModel tableModel, Map<String, Object> rowData, Long id) {
return UpdateTableDMLUtil.buildUpdateSql(tableModel, rowData, id);
}
/**
* 构建删除数据的SQL语句,根据id删除
*/
public static String buildDeleteSql(CustomTableDefinitionModel tableModel, Long id) {
return DeleteTableDmlUtil.buildDeleteSql(tableModel, id);
}
/**
* UNIQUE INDEX 的sql创建
*
* @param database
* @param table
* @param fieldName
* @return
*/
public static String buildCreateUniqueIndexSql(String database, String table, String fieldName) {
String indexName = "uk_" + fieldName;
String sql = "ALTER TABLE `" + database + "`.`" + table +
"` ADD INDEX " + indexName + "(`" + fieldName + "`) COMMENT '唯一索引'";
return sql;
}
/**
* 将字段类型转换为Doris数据类型
*
* @param fieldType 字段类型
* @return Doris数据类型
*/
public static String convertToFieldType(Integer fieldType) {
return convertToDorisTypeDefinition(fieldType);
}
/**
* 生成Doris ALTER TABLE SQL语句
*/
public static List<String> generateDorisAlterSqls(CustomTableDefinitionModel tableModel,
List<CustomFieldDefinitionModel> fieldsToAdd,
List<CustomFieldDefinitionModel> fieldsToUpdate,
Map<String, CustomFieldDefinitionModel> existingFieldMap) {
List<String> sqls = new ArrayList<>();
String tableName = "`" + tableModel.getDorisDatabase() + "`.`" + tableModel.getDorisTable() + "`";
// 添加新字段
for (CustomFieldDefinitionModel field : fieldsToAdd) {
// 字段是否允许为空,读取变量 DorisConfigContants.FIXED_FIELD_NULLABLE
boolean isNotNull = DorisConfigContants.FIXED_FIELD_NULLABLE ? false
: field.getNullableFlag() != null && field.getNullableFlag() == -1;
boolean hasDefault = field.getDefaultValue() != null && StringUtils.hasText(field.getDefaultValue());
String defaultClause = "";
if (!TableFieldTypeEnum.MEDIUMTEXT.getCode().equals(field.getFieldType()) && hasDefault) {
String formattedDefault = formatDefaultValue(field.getDefaultValue(), field.getFieldType());
if (formattedDefault != null) {
defaultClause = "DEFAULT " + formattedDefault;
}
}
String dorisType = convertToFieldType(field.getFieldType());
sqls.add(String.format("ALTER TABLE %s ADD COLUMN %s %s %s %s COMMENT '%s'",
tableName,
"`" + field.getFieldName() + "`",
dorisType,
isNotNull ? "NOT NULL" : "NULL",
defaultClause,
field.getFieldDescription() != null ? field.getFieldDescription() : ""));
// 如果新字段需要唯一索引
if (field.getUniqueFlag() != null && field.getUniqueFlag() == 1) {
String indexName = "uk_" + field.getFieldName();
sqls.add(String.format("ALTER TABLE %s ADD UNIQUE INDEX %s (`%s`) COMMENT '唯一索引'",
tableName, indexName, field.getFieldName()));
}
}
// 修改现有字段 (主要处理类型变更,约束变更在 validateFieldConstraintChanges 中已阻止)
for (CustomFieldDefinitionModel field : fieldsToUpdate) {
CustomFieldDefinitionModel existingField = existingFieldMap.get(field.getFieldName());
// 且不是系统字段,通过名称来获取,如果 defaultTableFieldEnum 有值,表示是系统字段
DefaultTableFieldEnum defaultTableFieldEnum = DefaultTableFieldEnum
.getEnumByFieldName(field.getFieldName());
// 另外看系统的描述,默认值,字段类型,是否必填,有修改,则需要生成 ALTER TABLE 语句
// 1. 默认值
// 2. 字段类型
// 3. 是否必填
// 4. 字段描述
// 5. 字段名
// 6. 字段类型
var fieldChange = existingField != null && Objects.isNull(defaultTableFieldEnum)
&& (!Objects.equals(existingField.getDefaultValue(), field.getDefaultValue())
|| !Objects.equals(existingField.getFieldDescription(), field.getFieldDescription()));
if (existingField != null && Objects.isNull(defaultTableFieldEnum) && fieldChange) {
log.warn("Field [{}] type changed from {} to {}, generating MODIFY COLUMN; confirm Doris supports this without data loss",
field.getFieldName(), existingField.getFieldType(), field.getFieldType());
boolean isNotNull = DorisConfigContants.FIXED_FIELD_NULLABLE ? false
: field.getNullableFlag() != null && field.getNullableFlag() == -1;
boolean hasDefault = field.getDefaultValue() != null && StringUtils.hasText(field.getDefaultValue());
String defaultClause = "";
if (!TableFieldTypeEnum.MEDIUMTEXT.getCode().equals(field.getFieldType()) && hasDefault) {
String formattedDefault = formatDefaultValue(field.getDefaultValue(), field.getFieldType());
if (formattedDefault != null) {
defaultClause = "DEFAULT " + formattedDefault;
}
}
String dorisType = convertToFieldType(field.getFieldType());
sqls.add(String.format("ALTER TABLE %s MODIFY COLUMN %s %s %s %s COMMENT '%s'",
tableName,
"`" + field.getFieldName() + "`",
dorisType,
isNotNull ? "NOT NULL" : "NULL",
defaultClause,
field.getFieldDescription() != null ? field.getFieldDescription() : ""));
}
// Note: 修改字段的其他属性如 COMMENT 也可以在这里处理
}
return sqls;
}
}

View File

@@ -0,0 +1,272 @@
package com.xspaceagi.compose.domain.util;
import java.math.BigDecimal;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.sdk.enums.DefaultTableFieldEnum;
import com.xspaceagi.compose.sdk.enums.TableFieldTypeEnum;
import com.xspaceagi.system.spec.exception.BizExceptionCodeEnum;
import com.xspaceagi.system.spec.exception.ComposeException;
import lombok.extern.slf4j.Slf4j;
/**
* Excel数据校验工具类
*/
@Slf4j
public final class ExcelDataValidatorUtil {
private ExcelDataValidatorUtil() {
}
// 数据类型限制常量(不暴露具体的数据库技术细节)
private static final int INT_MIN_VALUE = -2147483648;
private static final int INT_MAX_VALUE = 2147483647;
private static final int SHORT_TEXT_MAX_LENGTH = 255;
private static final int LONG_TEXT_MAX_LENGTH = 16777215; // 16MB
private static final int DECIMAL_MAX_PRECISION = 28;
private static final int DECIMAL_MAX_SCALE = 6;
/**
* 批量验证Excel数据的合规性
*
* @param tableModel 表定义模型
* @param excelData Excel数据列表
* @throws ComposeException 当数据不符合要求时抛出异常
*/
public static void validateExcelData(CustomTableDefinitionModel tableModel, List<Map<String, Object>> excelData) {
if (excelData == null || excelData.isEmpty()) {
return;
}
// 构建字段类型映射
Map<String, CustomFieldDefinitionModel> fieldDefinitionMap = null;
if (tableModel.getFieldList() != null) {
fieldDefinitionMap = tableModel.getFieldList().stream()
.filter(f -> f != null && f.getFieldName() != null)
.collect(Collectors.toMap(CustomFieldDefinitionModel::getFieldName, f -> f, (a, b) -> a));
}
// 逐行验证数据
for (int rowIndex = 0; rowIndex < excelData.size(); rowIndex++) {
Map<String, Object> rowData = excelData.get(rowIndex);
if (rowData == null || rowData.isEmpty()) {
continue;
}
validateRowData(tableModel, rowData, fieldDefinitionMap, rowIndex + 1);
}
log.info("Excel validation done, {} rows checked", excelData.size());
}
/**
* 验证单行数据
*
* @param tableModel 表定义模型
* @param rowData 行数据
* @param fieldDefinitionMap 字段定义映射
* @param rowNumber 行号(用于错误提示)
*/
private static void validateRowData(CustomTableDefinitionModel tableModel, Map<String, Object> rowData,
Map<String, CustomFieldDefinitionModel> fieldDefinitionMap, int rowNumber) {
for (Map.Entry<String, Object> entry : rowData.entrySet()) {
String fieldName = entry.getKey();
Object value = entry.getValue();
// 跳过系统字段的校验
if (isSystemField(fieldName)) {
continue;
}
// 获取字段定义信息
CustomFieldDefinitionModel fieldDefinition = null;
if (fieldDefinitionMap != null && fieldDefinitionMap.containsKey(fieldName)) {
fieldDefinition = fieldDefinitionMap.get(fieldName);
}
validateFieldValue(fieldName, value, fieldDefinition, rowNumber);
}
}
/**
* 验证单个字段值
*
* @param fieldName 字段名
* @param value 字段值
* @param fieldDefinition 字段定义信息
* @param rowNumber 行号
*/
private static void validateFieldValue(String fieldName, Object value, CustomFieldDefinitionModel fieldDefinition,
int rowNumber) {
// 空值检查
boolean isValueEmpty = (value == null) || (value instanceof String && ((String) value).trim().isEmpty());
if (isValueEmpty) {
// 校验必填字段(仅校验已启用的字段)
if (fieldDefinition != null
&& fieldDefinition.getEnabledFlag() != null && fieldDefinition.getEnabledFlag() == 1
&& fieldDefinition.getNullableFlag() != null && fieldDefinition.getNullableFlag() == -1) {
String errorMessage = String.format("第%d行字段'%s'为必填项,不能为空", rowNumber, fieldName);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
return;
}
// 根据字段类型进行校验
if (fieldDefinition == null) {
return;
}
Integer fieldType = fieldDefinition.getFieldType();
if (fieldType == null) {
return;
}
try {
if (TableFieldTypeEnum.BOOLEAN.getCode().equals(fieldType)) {
validateBooleanField(fieldName, value, rowNumber);
} else if (TableFieldTypeEnum.INTEGER.getCode().equals(fieldType)) {
validateIntegerField(fieldName, value, rowNumber);
} else if (TableFieldTypeEnum.NUMBER.getCode().equals(fieldType)) {
validateNumberField(fieldName, value, rowNumber);
} else if (TableFieldTypeEnum.STRING.getCode().equals(fieldType)) {
validateStringField(fieldName, value, rowNumber, SHORT_TEXT_MAX_LENGTH);
} else if (TableFieldTypeEnum.MEDIUMTEXT.getCode().equals(fieldType)) {
validateStringField(fieldName, value, rowNumber, LONG_TEXT_MAX_LENGTH);
} else if (TableFieldTypeEnum.DATE.getCode().equals(fieldType)) {
validateDateField(fieldName, value, rowNumber);
}
} catch (Exception e) {
log.error("Field validation failed, name: {}, value: {}, row: {}", fieldName, value, rowNumber, e);
throw e;
}
}
/**
* 校验布尔类型字段
*/
private static void validateBooleanField(String fieldName, Object value, int rowNumber) {
String stringValue = value.toString().trim();
if (!"true".equalsIgnoreCase(stringValue) && !"false".equalsIgnoreCase(stringValue)
&& !"1".equals(stringValue) && !"0".equals(stringValue)) {
String errorMessage = String.format("第%d行字段'%s'应填写true/false或1/0",
rowNumber, fieldName);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
}
/**
* 校验整数类型字段
*/
private static void validateIntegerField(String fieldName, Object value, int rowNumber) {
try {
if (value instanceof Number) {
// 如果是数值类型,检查范围
long longValue = ((Number) value).longValue();
if (longValue < INT_MIN_VALUE || longValue > INT_MAX_VALUE) {
String errorMessage = String.format("第%d行字段'%s'数值过大,请填写%d到%d之间的整数",
rowNumber, fieldName, INT_MIN_VALUE, INT_MAX_VALUE);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
} else {
// 如果是字符串类型,尝试转换并检查范围
String stringValue = value.toString().trim();
long longValue = Long.parseLong(stringValue);
if (longValue < INT_MIN_VALUE || longValue > INT_MAX_VALUE) {
String errorMessage = String.format("第%d行字段'%s'数值过大,请填写%d到%d之间的整数",
rowNumber, fieldName, INT_MIN_VALUE, INT_MAX_VALUE);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
}
} catch (NumberFormatException e) {
String errorMessage = String.format("第%d行字段'%s'请填写有效的整数",
rowNumber, fieldName);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
}
/**
* 校验数值类型字段
*/
private static void validateNumberField(String fieldName, Object value, int rowNumber) {
try {
BigDecimal decimal;
if (value instanceof Number) {
decimal = new BigDecimal(value.toString());
} else {
String stringValue = value.toString().trim();
decimal = new BigDecimal(stringValue);
}
// 检查总精度和小数位数
int precision = decimal.precision();
int scale = decimal.scale();
if (precision > DECIMAL_MAX_PRECISION) {
String errorMessage = String.format("第%d行字段'%s'数字太长,请限制在%d位以内",
rowNumber, fieldName, DECIMAL_MAX_PRECISION);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
if (scale > DECIMAL_MAX_SCALE) {
String errorMessage = String.format("第%d行字段'%s'小数位过多,请限制在%d位以内",
rowNumber, fieldName, DECIMAL_MAX_SCALE);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
} catch (NumberFormatException e) {
String errorMessage = String.format("第%d行字段'%s'请填写有效的数字",
rowNumber, fieldName);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
}
/**
* 校验字符串类型字段
*/
private static void validateStringField(String fieldName, Object value, int rowNumber, int maxLength) {
String stringValue = value.toString();
if (stringValue.length() > maxLength) {
String errorMessage = String.format("第%d行字段'%s'内容过长,请控制在%d个字符以内",
rowNumber, fieldName, maxLength);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
}
/**
* 校验日期类型字段
*/
private static void validateDateField(String fieldName, Object value, int rowNumber) {
if (!(value instanceof String)) {
String errorMessage = String.format("第%d行字段'%s'请填写正确的日期格式",
rowNumber, fieldName);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
try {
String dateStr = (String) value;
OffsetDateTime.parse(dateStr, DateTimeFormatter.ISO_OFFSET_DATE_TIME);
} catch (Exception e) {
String errorMessage = String.format("第%d行字段'%s'日期格式错误,请使用标准格式",
rowNumber, fieldName);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
}
/**
* 判断是否为系统字段
*/
private static boolean isSystemField(String fieldName) {
return DefaultTableFieldEnum.getEnumByFieldName(fieldName) != null;
}
}

View File

@@ -0,0 +1,482 @@
package com.xspaceagi.compose.domain.util;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.domain.util.dml.DeleteTableDmlUtil;
import com.xspaceagi.compose.domain.util.dml.InsertTableDMLUtil;
import com.xspaceagi.compose.domain.util.dml.UpdateTableDMLUtil;
import com.xspaceagi.compose.sdk.enums.DefaultTableFieldEnum;
import com.xspaceagi.compose.sdk.enums.TableFieldTypeEnum;
import com.xspaceagi.compose.spec.constants.DorisConfigContants;
import com.xspaceagi.system.spec.exception.BizExceptionCodeEnum;
import com.xspaceagi.system.spec.exception.ComposeException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* MySQL DDL语句生成工具类
*/
@Slf4j
@Component
public class MysqlTableDdlUtil {
/**
* 构建创建表的SQL语句
*
* @param tableModel 表定义模型
* @param fields 字段定义列表
* @return CREATE TABLE SQL 语句
*/
public static String buildCreateTableSql(CustomTableDefinitionModel tableModel,
List<CustomFieldDefinitionModel> fields) {
if (tableModel == null || !StringUtils.hasText(tableModel.getDorisDatabase())
|| !StringUtils.hasText(tableModel.getDorisTable())) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "表定义、数据库名或表名不能为空");
}
if (CollectionUtils.isEmpty(fields)) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "创建表时至少需要一个字段");
}
StringBuilder sql = new StringBuilder();
// 1. CREATE TABLE 语句头
sql.append("CREATE TABLE IF NOT EXISTS `")
.append(tableModel.getDorisDatabase()).append("`.`")
.append(tableModel.getDorisTable()).append("` (\n");
// 2. 添加 id 主键字段
sql.append(" `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT '主键ID',\n");
// 3. 字段定义
String columnDefinitions = fields.stream()
// 过滤掉id字段
.filter(f -> !f.getFieldName().equals("id"))
.map(MysqlTableDdlUtil::buildColumnDefinitionSql)
.collect(Collectors.joining(",\n"));
sql.append(columnDefinitions).append(",\n");
// 4. 添加主键约束
sql.append(" PRIMARY KEY (`id`)\n");
// 5. 表属性
sql.append(") ENGINE=InnoDB\n")
.append("DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci\n")
.append("COMMENT='").append(BuildSqlUtil.escapeSqlString(tableModel.getTableDescription())).append("'");
log.debug("Built CREATE TABLE SQL for {}.{}: \n{}", tableModel.getDorisDatabase(), tableModel.getDorisTable(),
sql.toString());
return sql.toString();
}
/**
* 构建单个字段的 SQL 定义部分
*/
private static String buildColumnDefinitionSql(CustomFieldDefinitionModel field) {
if (field == null || !StringUtils.hasText(field.getFieldName())) {
log.warn("Skipping invalid field definition");
return "";
}
StringBuilder fieldSql = new StringBuilder();
// 字段名
fieldSql.append(" `").append(field.getFieldName()).append("` ");
// 特殊处理 modified 字段
if ("modified".equalsIgnoreCase(field.getFieldName())) {
fieldSql.append("DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP");
} else {
// 字段类型
fieldSql.append(convertToMysqlTypeDefinition(field.getFieldType()));
// 标记是否为 NOT NULL,读取控制字段: DorisConfigContants.FIXED_FIELD_NULLABLE
boolean isNotNull = field.getNullableFlag() != null && field.getNullableFlag() == -1;
if (DorisConfigContants.FIXED_FIELD_NULLABLE) {
isNotNull = false;
}
// 非空约束
if (isNotNull) {
fieldSql.append(" NOT NULL");
} else {
fieldSql.append(" NULL");
}
// 默认值处理
if (!TableFieldTypeEnum.MEDIUMTEXT.getCode().equals(field.getFieldType())) {
if (field.getDefaultValue() != null) {
String formattedDefault = formatDefaultValue(field.getDefaultValue(), field.getFieldType());
if (formattedDefault != null && (!isNotNull || !"NULL".equals(formattedDefault))) {
fieldSql.append(" DEFAULT ").append(formattedDefault);
} else if (formattedDefault != null && isNotNull && "NULL".equals(formattedDefault)) {
log.warn("Field '{}' is NOT NULL but default resolves to NULL; no DEFAULT clause will be added.", field.getFieldName());
}
}
}
}
// 字段注释
if (StringUtils.hasText(field.getFieldDescription())) {
fieldSql.append(" COMMENT '").append(BuildSqlUtil.escapeSqlString(field.getFieldDescription())).append("'");
}
return fieldSql.toString();
}
/**
* 将内部字段类型代码转换为MySQL数据类型定义字符串
*/
private static String convertToMysqlTypeDefinition(Integer fieldTypeCode) {
if (fieldTypeCode == null) {
log.warn("Field type code is null, returning default definition: {}", TableFieldTypeEnum.STRING.getMysqlDefinition());
return TableFieldTypeEnum.STRING.getMysqlDefinition();
}
// MySQL的类型定义与Doris略有不同
for (TableFieldTypeEnum typeEnum : TableFieldTypeEnum.values()) {
if (typeEnum.getCode().equals(fieldTypeCode)) {
switch (typeEnum) {
case STRING:
return typeEnum.getMysqlDefinition();
case INTEGER:
return typeEnum.getMysqlDefinition();
case NUMBER:
return typeEnum.getMysqlDefinition();
case BOOLEAN:
return typeEnum.getMysqlDefinition();
case DATE:
return typeEnum.getMysqlDefinition();
case MEDIUMTEXT:
return typeEnum.getMysqlDefinition();
default:
return typeEnum.getMysqlDefinition();
}
}
}
return TableFieldTypeEnum.STRING.getMysqlDefinition();
}
/**
* 格式化 SQL 的默认值
*/
private static String formatDefaultValue(String defaultValue, Integer fieldTypeCode) {
if (!StringUtils.hasText(defaultValue) || "NULL".equalsIgnoreCase(defaultValue.trim())) {
return "NULL";
}
TableFieldTypeEnum typeEnum = null;
for (TableFieldTypeEnum t : TableFieldTypeEnum.values()) {
if (t.getCode().equals(fieldTypeCode)) {
typeEnum = t;
break;
}
}
if (typeEnum == null) {
log.warn("Unknown field type code: {}, treat as string", fieldTypeCode);
typeEnum = TableFieldTypeEnum.STRING;
}
switch (typeEnum) {
case INTEGER:
case NUMBER:
try {
Double.parseDouble(defaultValue);
return defaultValue;
} catch (NumberFormatException e) {
log.warn("Field is numeric but default '{}' is invalid, treating as string", defaultValue);
return "'" + BuildSqlUtil.escapeSqlString(defaultValue) + "'";
}
case BOOLEAN:
if ("true".equalsIgnoreCase(defaultValue) || "1".equals(defaultValue))
return "1";
if ("false".equalsIgnoreCase(defaultValue) || "0".equals(defaultValue))
return "0";
log.warn("Field is boolean but default '{}' is invalid, using '0'", defaultValue);
return "0";
case DATE:
if ("CURRENT_TIMESTAMP".equalsIgnoreCase(defaultValue.trim())) {
return "CURRENT_TIMESTAMP";
}
log.warn("Date field: default '{}' invalid; no DEFAULT clause.", defaultValue);
return null;
case MEDIUMTEXT:
return "'" + BuildSqlUtil.escapeSqlString(defaultValue) + "'";
case STRING:
default:
return "'" + BuildSqlUtil.escapeSqlString(defaultValue) + "'";
}
}
/**
* 生成修改表的SQL语句列表
*/
public static List<String> buildAlterTableSqls(String database, String table,
List<CustomFieldDefinitionModel> oldFields,
List<CustomFieldDefinitionModel> newFields) {
List<String> alterStatements = new ArrayList<>();
if (!StringUtils.hasText(database) || !StringUtils.hasText(table)) {
log.error("Database name or table name cannot be empty");
return alterStatements;
}
if (newFields == null)
newFields = new ArrayList<>();
if (oldFields == null)
oldFields = new ArrayList<>();
Map<String, CustomFieldDefinitionModel> oldFieldMap = oldFields.stream()
.filter(f -> f != null && StringUtils.hasText(f.getFieldName()))
.collect(Collectors.toMap(CustomFieldDefinitionModel::getFieldName, Function.identity()));
Map<String, CustomFieldDefinitionModel> newFieldMap = newFields.stream()
.filter(f -> f != null && StringUtils.hasText(f.getFieldName()))
.collect(Collectors.toMap(CustomFieldDefinitionModel::getFieldName, Function.identity()));
// 处理新增字段
for (CustomFieldDefinitionModel newField : newFields) {
if (!oldFieldMap.containsKey(newField.getFieldName())) {
String addColumnSql = buildAddColumnSql(newField);
if (StringUtils.hasText(addColumnSql)) {
alterStatements.add(String.format("ALTER TABLE `%s`.`%s` ADD COLUMN %s",
database, table, addColumnSql));
}
}
}
// 处理修改字段
for (Map.Entry<String, CustomFieldDefinitionModel> entry : newFieldMap.entrySet()) {
String fieldName = entry.getKey();
CustomFieldDefinitionModel newField = entry.getValue();
CustomFieldDefinitionModel oldField = oldFieldMap.get(fieldName);
if (oldField != null && needsModification(oldField, newField)) {
String modifyColumnSql = buildModifyColumnSql(newField);
if (StringUtils.hasText(modifyColumnSql)) {
alterStatements.add(String.format("ALTER TABLE `%s`.`%s` MODIFY COLUMN %s",
database, table, modifyColumnSql));
}
}
}
return alterStatements;
}
/**
* 判断字段是否需要修改
*/
private static boolean needsModification(CustomFieldDefinitionModel oldField, CustomFieldDefinitionModel newField) {
return !Objects.equals(oldField.getFieldType(), newField.getFieldType()) ||
!Objects.equals(oldField.getNullableFlag(), newField.getNullableFlag()) ||
!Objects.equals(oldField.getDefaultValue(), newField.getDefaultValue()) ||
!Objects.equals(oldField.getFieldDescription(), newField.getFieldDescription());
}
/**
* 构建添加列的SQL
*/
private static String buildAddColumnSql(CustomFieldDefinitionModel field) {
return buildColumnDefinitionSql(field);
}
/**
* 构建修改列的SQL
*/
private static String buildModifyColumnSql(CustomFieldDefinitionModel field) {
return buildColumnDefinitionSql(field);
}
/**
* 构建创建数据库的SQL语句
*/
public static String buildCreateDatabaseSql(String database) {
return String.format(
"CREATE DATABASE IF NOT EXISTS `%s` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci",
BuildSqlUtil.escapeSqlString(database));
}
/**
* 构建删除数据库的SQL语句
*/
public static String buildDropDatabaseSql(String database) {
return String.format("DROP DATABASE IF EXISTS `%s`", BuildSqlUtil.escapeSqlString(database));
}
/**
* 构建删除表的SQL语句
*/
public static String buildDropTableSql(String database, String table) {
return String.format("DROP TABLE IF EXISTS `%s`.`%s`",
BuildSqlUtil.escapeSqlString(database), BuildSqlUtil.escapeSqlString(table));
}
/**
* 获取表的建表DDL语句
*
* @param database 数据库名
* @param table 表名
* @return 建表DDL语句
*/
public static String getCreateTableDdl(String database, String table) {
if (!StringUtils.hasText(database) || !StringUtils.hasText(table)) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "数据库名或表名不能为空");
}
return "SHOW CREATE TABLE `" + database + "`.`" + table + "`";
}
/**
* 获取检查表是否存在的SQL语句
*
* @param database 数据库名
* @param table 表名
* @return 检查表是否存在的SQL语句
*/
public static String getTableExistsSql(String database, String table) {
if (!StringUtils.hasText(database) || !StringUtils.hasText(table)) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "数据库名或表名不能为空");
}
return "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '"
+ BuildSqlUtil.escapeSqlString(database) +
"' AND table_name = '" + BuildSqlUtil.escapeSqlString(table) + "'";
}
/**
* 获取查询表总记录数的SQL语句
*
* @param database 数据库名
* @param table 表名
* @return 查询表总记录数的SQL语句
*/
public static String getTableCountSql(String database, String table) {
if (!StringUtils.hasText(database) || !StringUtils.hasText(table)) {
throw ComposeException.build(BizExceptionCodeEnum.resourceDataNotFound, "数据库名或表名不能为空");
}
return "SELECT COUNT(*) FROM `" + BuildSqlUtil.escapeSqlString(database) + "`.`"
+ BuildSqlUtil.escapeSqlString(table) + "`";
}
/**
* 构建插入数据的SQL语句 (使用 INSERT INTO ... SET ... 语法)
*/
public static String buildInsertSql(CustomTableDefinitionModel tableModel, Map<String, Object> rowData) {
return InsertTableDMLUtil.buildMysqlInsertSql(tableModel, rowData);
}
/**
* 构建更新数据的SQL语句,根据id来修改行数
*/
public static String buildUpdateSql(CustomTableDefinitionModel tableModel, Map<String, Object> rowData, Long id) {
return UpdateTableDMLUtil.buildUpdateSql(tableModel, rowData, id);
}
/**
* 构建删除数据的SQL语句,根据id删除
*/
public static String buildDeleteSql(CustomTableDefinitionModel tableModel, Long id) {
return DeleteTableDmlUtil.buildDeleteSql(tableModel, id);
}
public static String buildCreateUniqueIndexSql(String database, String table, String fieldName) {
String indexName = "uk_" + fieldName;
return String.format("ALTER TABLE `%s`.`%s` ADD UNIQUE INDEX %s(`%s`);", database, table, indexName, fieldName);
}
/**
* 将字段类型转换为Mysql数据类型
*
* @param fieldType 字段类型
* @return Mysql数据类型
*/
public static String convertToFieldType(Integer fieldType) {
return convertToMysqlTypeDefinition(fieldType);
}
/**
* 生成Doris ALTER TABLE SQL语句
*/
public static List<String> generateDorisAlterSqls(CustomTableDefinitionModel tableModel,
List<CustomFieldDefinitionModel> fieldsToAdd,
List<CustomFieldDefinitionModel> fieldsToUpdate,
Map<String, CustomFieldDefinitionModel> existingFieldMap) {
List<String> sqls = new ArrayList<>();
String tableName = "`" + tableModel.getDorisDatabase() + "`.`" + tableModel.getDorisTable() + "`";
// 添加新字段
for (CustomFieldDefinitionModel field : fieldsToAdd) {
// 字段是否允许为空,读取变量 DorisConfigContants.FIXED_FIELD_NULLABLE
boolean isNotNull = DorisConfigContants.FIXED_FIELD_NULLABLE ? false
: field.getNullableFlag() != null && field.getNullableFlag() == -1;
boolean hasDefault = field.getDefaultValue() != null && StringUtils.hasText(field.getDefaultValue());
String defaultClause = "";
if (!TableFieldTypeEnum.MEDIUMTEXT.getCode().equals(field.getFieldType()) && hasDefault) {
String formattedDefault = formatDefaultValue(field.getDefaultValue(), field.getFieldType());
if (formattedDefault != null) {
defaultClause = "DEFAULT " + formattedDefault;
}
}
String dorisType = convertToFieldType(field.getFieldType());
sqls.add(String.format("ALTER TABLE %s ADD COLUMN %s %s %s %s COMMENT '%s'",
tableName,
"`" + field.getFieldName() + "`",
dorisType,
isNotNull ? "NOT NULL" : "NULL",
defaultClause,
field.getFieldDescription() != null ? field.getFieldDescription() : ""));
// 如果新字段需要唯一索引
if (field.getUniqueFlag() != null && field.getUniqueFlag() == 1) {
String indexName = "uk_" + field.getFieldName();
sqls.add(String.format("ALTER TABLE %s ADD UNIQUE INDEX %s (`%s`) COMMENT '唯一索引'",
tableName, indexName, field.getFieldName()));
}
}
// 修改现有字段 (主要处理类型变更,约束变更在 validateFieldConstraintChanges 中已阻止)
for (CustomFieldDefinitionModel field : fieldsToUpdate) {
CustomFieldDefinitionModel existingField = existingFieldMap.get(field.getFieldName());
// 且不是系统字段,通过名称来获取,如果 defaultTableFieldEnum 有值,表示是系统字段
DefaultTableFieldEnum defaultTableFieldEnum = DefaultTableFieldEnum
.getEnumByFieldName(field.getFieldName());
// 另外看系统的描述,默认值,字段类型,是否必填,有修改,则需要生成 ALTER TABLE 语句
// 1. 默认值
// 2. 字段类型
// 3. 是否必填
// 4. 字段描述
// 5. 字段名
// 6. 字段类型
var fieldChange = existingField != null && Objects.isNull(defaultTableFieldEnum)
&& (!Objects.equals(existingField.getDefaultValue(), field.getDefaultValue())
|| !Objects.equals(existingField.getFieldDescription(), field.getFieldDescription()));
if (existingField != null && Objects.isNull(defaultTableFieldEnum) && fieldChange) {
log.warn("Field [{}] type changed from {} to {}, generating MODIFY COLUMN; confirm Doris supports this without data loss",
field.getFieldName(), existingField.getFieldType(), field.getFieldType());
boolean isNotNull = DorisConfigContants.FIXED_FIELD_NULLABLE ? false
: field.getNullableFlag() != null && field.getNullableFlag() == -1;
boolean hasDefault = field.getDefaultValue() != null && StringUtils.hasText(field.getDefaultValue());
String defaultClause = "";
if (!TableFieldTypeEnum.MEDIUMTEXT.getCode().equals(field.getFieldType()) && hasDefault) {
String formattedDefault = formatDefaultValue(field.getDefaultValue(), field.getFieldType());
if (formattedDefault != null) {
defaultClause = "DEFAULT " + formattedDefault;
}
}
String dorisType = convertToFieldType(field.getFieldType());
sqls.add(String.format("ALTER TABLE %s MODIFY COLUMN %s %s %s %s COMMENT '%s'",
tableName,
"`" + field.getFieldName() + "`",
dorisType,
isNotNull ? "NOT NULL" : "NULL",
defaultClause,
field.getFieldDescription() != null ? field.getFieldDescription() : ""));
}
// Note: 修改字段的其他属性如 COMMENT 也可以在这里处理
}
return sqls;
}
}

View File

@@ -0,0 +1,138 @@
package com.xspaceagi.compose.domain.util;
import cn.hutool.core.lang.Snowflake;
import cn.hutool.core.util.IdUtil;
import com.xspaceagi.system.sdk.service.AbstractTaskExecuteService;
import com.xspaceagi.system.sdk.service.ScheduleTaskApiService;
import com.xspaceagi.system.sdk.service.dto.ScheduleTaskDto;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* 雪花算法的工具类,用于生成主键id,给doris设置主键id使用
*/
@Component("snowflakeIdWorker")
@Slf4j
public class SnowflakeIdWorker extends AbstractTaskExecuteService {
private static Snowflake snowflake;
private static Long workerId = null;
@Resource
private StringRedisTemplate stringRedisTemplate;
private static final String REDIS_WORKER_ID_KEY_PREFIX = "snowflake:worker:id:";
private static final int MAX_WORKER_ID = 31; // 支持 0~31 共32个节点
private static final long DATACENTER_ID = 1L;
private static final long WORKER_ID_EXPIRE_SECONDS = 180; // 3分钟
@Resource
private ScheduleTaskApiService scheduleTaskApiService;
@PostConstruct
public void init() {
try {
for (int i = 0; i <= MAX_WORKER_ID; i++) {
Boolean success = null;
try {
success = stringRedisTemplate.opsForValue().setIfAbsent(
REDIS_WORKER_ID_KEY_PREFIX + i,
"1",
WORKER_ID_EXPIRE_SECONDS,
TimeUnit.SECONDS);
} catch (Exception e) {
// Redis 不可用,直接抛出异常,应用拒绝启动
log.error("[SnowflakeIdWorker] Redis workerId allocation failed", e);
throw new RuntimeException("Redis 分配 workerId 失败", e);
}
if (Boolean.TRUE.equals(success)) {
workerId = (long) i;
break;
}
}
if (workerId == null) {
throw new RuntimeException("没有可用的workerId请检查是否有节点未释放workerId或增大MAX_WORKER_ID");
}
// 定时续约可选防止服务长时间运行workerId过期
// 你可以用定时任务每隔一段时间刷新自己的workerId过期时间
snowflake = IdUtil.getSnowflake(workerId, DATACENTER_ID);
} catch (Exception e) {
// 启动时分配失败,应用拒绝启动
log.error("[SnowflakeIdWorker] init failed", e);
throw e;
}
scheduleTaskApiService.start(ScheduleTaskDto.builder()
.taskId("snowflakeIdWorker")
.beanId("snowflakeIdWorker")
.maxExecTimes(Long.MAX_VALUE)
.cron(ScheduleTaskDto.Cron.EVERY_30_SECOND.getCron())
.params(Map.of())
.build());
}
@Override
protected boolean execute(ScheduleTaskDto scheduleTaskDto) {
try {
refreshWorkerIdExpire();
} catch (Exception e) {
log.error("[SnowflakeIdWorker] heartbeat renew failed", e);
}
return false;
}
public void refreshWorkerIdExpire() {
if (workerId != null) {
try {
stringRedisTemplate.expire(
REDIS_WORKER_ID_KEY_PREFIX + workerId,
WORKER_ID_EXPIRE_SECONDS,
TimeUnit.SECONDS);
} catch (Exception e) {
// 只记录日志,不抛出异常,防止定时任务线程崩溃
log.error("[SnowflakeIdWorker] heartbeat Redis failed", e);
// 可选:报警
}
}
}
// 优雅停机时主动释放workerId可选
@PreDestroy
public void releaseWorkerId() {
if (workerId != null) {
try {
stringRedisTemplate.delete(REDIS_WORKER_ID_KEY_PREFIX + workerId);
} catch (Exception e) {
// 只记录日志,不影响主流程
log.error("[SnowflakeIdWorker] release workerId", e);
}
}
}
/**
* 生成雪花算法ID
*
* @return 雪花算法ID
*/
public static long nextId() {
return snowflake.nextId();
}
/**
* 生成雪花算法ID字符串
*
* @return 雪花算法ID字符串
*/
public static String nextIdStr() {
return snowflake.nextIdStr();
}
}

View File

@@ -0,0 +1,362 @@
package com.xspaceagi.compose.domain.util;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.DoubleValue;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.LongValue;
import net.sf.jsqlparser.expression.StringValue;
import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.alter.Alter;
import net.sf.jsqlparser.statement.create.table.CreateTable;
import net.sf.jsqlparser.statement.delete.Delete;
import net.sf.jsqlparser.statement.drop.Drop;
import net.sf.jsqlparser.statement.insert.Insert;
import net.sf.jsqlparser.statement.select.PlainSelect;
import net.sf.jsqlparser.statement.select.Select;
import net.sf.jsqlparser.statement.truncate.Truncate;
import net.sf.jsqlparser.statement.update.Update;
import net.sf.jsqlparser.statement.select.FromItem;
import net.sf.jsqlparser.statement.select.Join;
import net.sf.jsqlparser.statement.select.LateralSubSelect;
import net.sf.jsqlparser.statement.select.ParenthesedFromItem;
import org.apache.commons.lang3.StringUtils;
import com.xspaceagi.compose.spec.constants.DorisConfigContants;
import com.xspaceagi.compose.spec.utils.ComposeExceptionUtils;
import com.xspaceagi.system.spec.exception.BizExceptionCodeEnum;
import com.xspaceagi.system.spec.exception.ComposeException;
import net.sf.jsqlparser.statement.select.Limit;
@Slf4j
public class SqlParserUtil {
public enum SqlType {
SELECT, // SELECT 查询语句
INSERT, // INSERT 插入语句
UPDATE, // UPDATE 更新语句
DELETE, // DELETE 删除语句
DDL // CREATE, ALTER, DROP, TRUNCATE 等DDL语句
}
/**
* 验证SQL语法和安全性
*/
public static void validateSql(String sql) throws JSQLParserException {
validateSql(sql, false);
}
/**
* 验证SQL语句
*
* @param sql SQL语句
* @param allowDDL 是否允许DDL语句
* @throws JSQLParserException SQL解析失败
* @throws IllegalArgumentException SQL验证失败
*/
public static void validateSql(String sql, boolean allowDDL) throws JSQLParserException {
if (StringUtils.isBlank(sql)) {
throw ComposeException.build(BizExceptionCodeEnum.composeSqlEmpty);
}
try {
Statement statement = CCJSqlParserUtil.parse(sql);
if (!allowDDL) {
// 只禁止 DDL 语句,允许 select、update、delete、insert
if (statement instanceof CreateTable
|| statement instanceof Alter
|| statement instanceof Drop
|| statement instanceof Truncate) {
throw ComposeException.build(BizExceptionCodeEnum.composeSqlOnlyDdl, sql);
}
}
} catch (JSQLParserException e) {
log.error("SQL parse failed: sql={}", sql, e);
throw e;
}
}
/**
* 解析SQL并返回SQL类型
*/
public static SqlType getSqlType(String sql) throws JSQLParserException {
Statement statement = CCJSqlParserUtil.parse(sql);
if (statement instanceof Select) {
return SqlType.SELECT;
} else if (statement instanceof Insert) {
return SqlType.INSERT;
} else if (statement instanceof Update) {
return SqlType.UPDATE;
} else if (statement instanceof Delete) {
return SqlType.DELETE;
} else if (statement instanceof CreateTable ||
statement instanceof Alter ||
statement instanceof Drop ||
statement instanceof Truncate) {
return SqlType.DDL;
}
throw ComposeException.build(BizExceptionCodeEnum.composeUnsupportedSqlType, statement.getClass().getSimpleName());
}
/**
* 修改SQL替换表名并添加限制条件默认不允许执行DDL语句
*/
public static String modifySql(String originalSql, String newTableName,
Map<String, Object> extArgs) throws JSQLParserException {
return modifySql(originalSql, newTableName, extArgs, false);
}
/**
* 修改SQL替换表名并添加限制条件
*
* @param originalSql 原始SQL
* @param newTableName 新表名
* @param extArgs 额外的限制条件
* @param allowDDL 是否允许执行DDL语句
* @return 修改后的SQL
* @throws JSQLParserException SQL解析异常
* @throws IllegalArgumentException SQL验证失败或不支持的SQL类型
*/
public static String modifySql(String originalSql, String newTableName,
Map<String, Object> extArgs, boolean allowDDL) throws JSQLParserException {
try {
// 验证SQL
validateSql(originalSql, allowDDL);
Statement statement = CCJSqlParserUtil.parse(originalSql);
// 根据SQL类型添加限制条件
SqlType sqlType = getSqlType(originalSql);
switch (sqlType) {
case SELECT:
Select select = (Select) statement;
PlainSelect plainSelect = select.getPlainSelect();
if (plainSelect != null) {
// 替换所有表名
replaceTableName(plainSelect.getFromItem(), newTableName);
if (plainSelect.getJoins() != null) {
for (Join join : plainSelect.getJoins()) {
replaceTableName(join.getRightItem(), newTableName);
}
}
// 添加限制条件
addSelectRestrictions(plainSelect, extArgs);
// 检查是否有 LIMIT如果没有则加上 LIMIT 1000
if (plainSelect.getLimit() == null) {
var queryLimit = DorisConfigContants.DEFAULT_QUERY_LIMIT;
var limit = new Limit().withRowCount(new LongValue(queryLimit));
plainSelect.setLimit(limit);
}
}
break;
case UPDATE:
if (statement instanceof Update update) {
// 替换表名
if (update.getTable() != null) {
update.getTable().setName(newTableName);
}
addUpdateRestrictions(update, extArgs);
}
break;
case DELETE:
if (statement instanceof Delete delete) {
// 替换表名
if (delete.getTable() != null) {
delete.getTable().setName(newTableName);
}
addDeleteRestrictions(delete, extArgs);
}
break;
case INSERT:
if (statement instanceof Insert insert) {
// 替换表名
if (insert.getTable() != null) {
insert.getTable().setName(newTableName);
}
// INSERT 不需要添加限制条件
}
break;
case DDL:
// DDL 语句不需要处理
break;
default:
throw ComposeException.build(BizExceptionCodeEnum.composeUnsupportedSqlType, sqlType.name());
}
return statement.toString();
} catch (JSQLParserException e) {
log.error("SQL parse failed: {}", originalSql, e);
throw e;
} catch (Exception e) {
log.error("SQL processing error: {}, cause: {}", originalSql, e.getMessage(), e);
throw e;
}
}
/**
* 为 SELECT 语句添加限制条件
*/
private static void addSelectRestrictions(PlainSelect plainSelect, Map<String, Object> extArgs) {
Expression where = plainSelect.getWhere();
Expression newWhere = buildRestrictions(where, extArgs);
plainSelect.setWhere(newWhere);
}
/**
* 为 UPDATE 语句添加限制条件
*/
private static void addUpdateRestrictions(Update update, Map<String, Object> extArgs) {
Expression where = update.getWhere();
Expression newWhere = buildRestrictions(where, extArgs);
update.setWhere(newWhere);
}
/**
* 为 DELETE 语句添加限制条件
*/
private static void addDeleteRestrictions(Delete delete, Map<String, Object> extArgs) {
Expression where = delete.getWhere();
Expression newWhere = buildRestrictions(where, extArgs);
delete.setWhere(newWhere);
}
/**
* 构建限制条件表达式
*/
private static Expression buildRestrictions(Expression originalWhere, Map<String, Object> extArgs) {
if (extArgs == null || extArgs.isEmpty()) {
return originalWhere;
}
List<Expression> conditions = new ArrayList<>();
// 遍历所有限制参数
for (Map.Entry<String, Object> entry : extArgs.entrySet()) {
String columnName = entry.getKey();
Object value = entry.getValue();
if (value != null) {
Expression condition = createEqualsExpression(columnName, value);
if (condition != null) {
conditions.add(condition);
}
}
}
// 合并条件
Expression newWhere = null;
for (Expression condition : conditions) {
if (newWhere == null) {
newWhere = condition;
} else {
newWhere = new AndExpression(newWhere, condition);
}
}
// 如果原始SQL有where条件则与新的条件合并
if (originalWhere != null && newWhere != null) {
return new AndExpression(originalWhere, newWhere);
}
return Objects.requireNonNullElse(newWhere, originalWhere);
}
/**
* 根据值的类型创建等于表达式
*
* @throws IllegalArgumentException 创建表达式失败
*/
private static Expression createEqualsExpression(String columnName, Object value) {
if (value == null) {
return null;
}
try {
if (value instanceof Number number) {
if (number instanceof Long || number instanceof Integer) {
return new EqualsTo(
new net.sf.jsqlparser.schema.Column(columnName),
new LongValue(number.longValue()));
} else if (number instanceof Double || number instanceof Float) {
return new EqualsTo(
new net.sf.jsqlparser.schema.Column(columnName),
new DoubleValue(String.valueOf(number)));
}
} else if (value instanceof String str) {
return new EqualsTo(
new net.sf.jsqlparser.schema.Column(columnName),
new StringValue(str));
} else if (value instanceof Boolean bool) {
return new EqualsTo(
new net.sf.jsqlparser.schema.Column(columnName),
new LongValue(bool ? 1L : 0L));
}
log.warn("Unsupported param type: column={}, value={}, type={}", columnName, value, value.getClass().getName());
return null;
} catch (Exception e) {
log.error("Build condition failed: column={}, value={}", columnName, value, e);
var errorMessage = ComposeExceptionUtils.getRootErrorMessage(e);
throw ComposeException.build(BizExceptionCodeEnum.composeBuildConditionFailed, errorMessage);
}
}
/**
* 验证SQL是否只包含一个表
*/
public static boolean isSingleTableQuery(String sql) throws JSQLParserException {
Statement statement = CCJSqlParserUtil.parse(sql);
if (statement instanceof Select) {
Select select = (Select) statement;
PlainSelect plainSelect = select.getPlainSelect();
if (plainSelect != null) {
FromItem fromItem = plainSelect.getFromItem();
return fromItem instanceof Table;
}
}
return true; // 非SELECT语句默认为单表操作
}
/**
* 获取SQL操作类型
*/
public static SqlType getSqlOperationType(String sql) throws JSQLParserException {
return getSqlType(sql);
}
/**
* 替换表名
*/
private static void replaceTableName(FromItem fromItem, String newTableName) {
if (fromItem instanceof Table table) {
table.setName(newTableName);
} else if (fromItem instanceof LateralSubSelect lateral) {
var select = lateral.getSelect();
if (select instanceof PlainSelect subPlainSelect) {
replaceTableName(subPlainSelect.getFromItem(), newTableName);
if (subPlainSelect.getJoins() != null) {
for (Join join : subPlainSelect.getJoins()) {
replaceTableName(join.getRightItem(), newTableName);
}
}
}
} else if (fromItem instanceof ParenthesedFromItem parenthesed) {
replaceTableName(parenthesed.getFromItem(), newTableName);
} else {
// 其他类型如 TableFunction、ValuesList 等,通常无需替换表名
log.warn("Unsupported fromItem type: {}", fromItem.getClass().getName());
}
}
}

View File

@@ -0,0 +1,299 @@
package com.xspaceagi.compose.domain.util;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import com.xspaceagi.compose.DatabaseTypeEnum;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.system.spec.exception.BizExceptionCodeEnum;
import com.xspaceagi.system.spec.exception.ComposeException;
import lombok.extern.slf4j.Slf4j;
/**
* tableDDlUtil的包装类,区分用户配置的 mysql,还是doris,使用对应的工具类
* <p>
* DorisTableDdlUtil , MysqlTableDdlUtil,根据配置,选择对应的工具类调用
* </p>
*/
@Slf4j
@Component
public class TableDbWrapperUtil {
private static final String DB_TYPE_MYSQL = DatabaseTypeEnum.MYSQL.getType();
private static final String DB_TYPE_DORIS = DatabaseTypeEnum.DORIS.getType();
@Value("${spring.datasource.sql-generator.type:mysql}")
private String dbType;
@Value("${spring.datasource.dynamic.datasource.doris.url}")
private String dorisUrl;
/**
* 获取Doris数据库名称
* 从数据源URL中解析数据库名称
*
* @return 数据库名称
*/
public String getDorisDatabase() {
if (!StringUtils.hasText(dorisUrl)) {
throw ComposeException.build(BizExceptionCodeEnum.composeDorisUrlNotConfigured);
}
try {
// 示例URL:
// jdbc:mysql://192.168.1.12:3306/agent_custom_table_test?serverTimezone=Asia/Shanghai...
// 1. 先用?分割获取前半部分
String urlWithoutParams = dorisUrl.split("\\?")[0];
// 2. 再用/分割获取最后一部分即数据库名
String[] parts = urlWithoutParams.split("/");
String databaseName = parts[parts.length - 1];
if (!StringUtils.hasText(databaseName)) {
log.error("Failed to parse DB name from URL[{}], URL: {}", dorisUrl);
throw ComposeException.build(BizExceptionCodeEnum.composeDorisDbNameParseFailed, "无法从URL中解析出数据库名称");
}
log.debug("Parsed DB name from URL[{}]: {}", dorisUrl, databaseName);
return databaseName;
} catch (Exception e) {
log.error("Parse Doris DB name failed, URL: {}", dorisUrl, e);
throw ComposeException.build(BizExceptionCodeEnum.composeDorisDbNameParseFailed, e.getMessage());
}
}
/**
* 构建创建数据库的SQL语句
*/
public String buildCreateDatabaseSql(String database) {
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.buildCreateDatabaseSql(database);
}
return MysqlTableDdlUtil.buildCreateDatabaseSql(database);
}
/**
* 构建删除数据库的SQL语句
*/
public String buildDropDatabaseSql(String database) {
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.buildDropDatabaseSql(database);
}
return MysqlTableDdlUtil.buildDropDatabaseSql(database);
}
/**
* 构建创建表的SQL语句
*/
public String buildCreateTableSql(CustomTableDefinitionModel tableModel, List<CustomFieldDefinitionModel> fields) {
validateDbType();
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.buildCreateTableSql(tableModel, fields);
}
return MysqlTableDdlUtil.buildCreateTableSql(tableModel, fields);
}
/**
* 构建删除表的SQL语句
*/
public String buildDropTableSql(String database, String table) {
validateDbType();
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.buildDropTableSql(database, table);
}
return MysqlTableDdlUtil.buildDropTableSql(database, table);
}
/**
* 生成修改表的SQL语句列表
*/
public List<String> buildAlterTableSqls(String database, String table,
List<CustomFieldDefinitionModel> oldFields,
List<CustomFieldDefinitionModel> newFields) {
validateDbType();
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.buildAlterTableSqls(database, table, oldFields, newFields);
}
return MysqlTableDdlUtil.buildAlterTableSqls(database, table, oldFields, newFields);
}
/**
* 获取表的建表DDL语句
*
* @param database 数据库名
* @param table 表名
* @return 建表DDL语句
*/
public String getShowCreateTableDdl(String database, String table) {
validateDbType();
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.getCreateTableDdl(database, table);
}
return MysqlTableDdlUtil.getCreateTableDdl(database, table);
}
/**
* 获取检查表是否存在的SQL语句
*
* @param database 数据库名
* @param table 表名
* @return 检查表是否存在的SQL语句
*/
public String getTableExistsSql(String database, String table) {
validateDbType();
String sql;
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
sql = DorisTableDdlUtil.getTableExistsSql(database, table);
} else {
sql = MysqlTableDdlUtil.getTableExistsSql(database, table);
}
log.debug("Check SQL exists, database: {}, table: {}, sql={}", database, table, sql);
return sql;
}
/**
* 获取查询表总记录数的SQL语句
*
* @param database 数据库名
* @param table 表名
* @return 查询表总记录数的SQL语句
*/
public String getTableCountSql(String database, String table) {
validateDbType();
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
log.debug("Count SQL for Doris, database: {}, table: {}", database, table);
return DorisTableDdlUtil.getTableCountSql(database, table);
}
log.debug("Count SQL for MySQL, database: {}, table: {}", database, table);
return MysqlTableDdlUtil.getTableCountSql(database, table);
}
/**
* 验证数据库类型配置是否有效
*/
private void validateDbType() {
if (!StringUtils.hasText(dbType)) {
log.error("DB type not configured, spring.datasource.sql-generator.type, dbType: {}", dbType);
throw ComposeException.build(BizExceptionCodeEnum.composeDbTypeNotConfigured);
}
if (!DB_TYPE_MYSQL.equalsIgnoreCase(dbType) && !DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
log.error("Unsupported DB type: {}, only mysql or doris", dbType);
throw ComposeException.build(BizExceptionCodeEnum.composeUnsupportedDbType, dbType);
}
}
/**
* 获取当前使用的数据库类型
*/
public String getCurrentDbType() {
return dbType;
}
/**
* 构建插入数据的SQL语句
*
* @param tableModel 表定义
* @param rowData 行数据
* @return 插入数据的SQL语句
*/
public String buildInsertSql(CustomTableDefinitionModel tableModel, Map<String, Object> rowData) {
validateDbType();
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.buildInsertSql(tableModel, rowData);
}
return MysqlTableDdlUtil.buildInsertSql(tableModel, rowData);
}
/**
* 构建更新数据的SQL语句,根据id来修改行数
*
* @param tableModel 表定义
* @param rowData 行数据
* @param id 行id
* @return 更新数据的SQL语句
*/
public String buildUpdateSql(CustomTableDefinitionModel tableModel, Map<String, Object> rowData, Long id) {
validateDbType();
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.buildUpdateSql(tableModel, rowData, id);
}
return MysqlTableDdlUtil.buildUpdateSql(tableModel, rowData, id);
}
/**
* 构建删除数据的SQL语句,根据id删除
*
* @param tableModel 表定义
* @param id 行id
* @return 删除数据的SQL语句
*/
public String buildDeleteSql(CustomTableDefinitionModel tableModel, Long id) {
validateDbType();
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.buildDeleteSql(tableModel, id);
}
return MysqlTableDdlUtil.buildDeleteSql(tableModel, id);
}
/**
* 构建创建唯一索引的SQL语句
*
* @param database 数据库名
* @param table 表名
* @param fieldName 字段名
* @return 创建唯一索引的SQL语句
*/
public String buildCreateUniqueIndexSql(String database, String table, String fieldName) {
validateDbType();
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.buildCreateUniqueIndexSql(database, table, fieldName);
}
return MysqlTableDdlUtil.buildCreateUniqueIndexSql(database, table, fieldName);
}
/**
* 将字段类型转换为Doris数据类型
*
* @param fieldType 字段类型
* @return Doris数据类型
*/
public String convertToFieldType(Integer fieldType) {
// 判断数据库是mysql,还是 doris
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.convertToFieldType(fieldType);
}
return MysqlTableDdlUtil.convertToFieldType(fieldType);
}
/**
* 生成 ALTER TABLE SQL语句
*
* @param tableModel
* @param fieldsToAdd
* @param fieldsToUpdate
* @param existingFieldMap
* @return
*/
public List<String> generateDorisAlterSqls(CustomTableDefinitionModel tableModel,
List<CustomFieldDefinitionModel> fieldsToAdd,
List<CustomFieldDefinitionModel> fieldsToUpdate,
Map<String, CustomFieldDefinitionModel> existingFieldMap) {
if (DB_TYPE_DORIS.equalsIgnoreCase(dbType)) {
return DorisTableDdlUtil.generateDorisAlterSqls(tableModel, fieldsToAdd, fieldsToUpdate, existingFieldMap);
}
return MysqlTableDdlUtil.generateDorisAlterSqls(tableModel, fieldsToAdd, fieldsToUpdate, existingFieldMap);
}
/**
* 获取数据库类型
*
* @return
*/
public String getDbType() {
return dbType;
}
}

View File

@@ -0,0 +1,30 @@
package com.xspaceagi.compose.domain.util.dml;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.domain.util.BuildSqlUtil;
import com.xspaceagi.system.spec.exception.BizExceptionCodeEnum;
import com.xspaceagi.system.spec.exception.ComposeException;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public final class DeleteTableDmlUtil {
private DeleteTableDmlUtil() {}
/**
* 构建删除数据的SQL语句,根据id删除
*/
public static String buildDeleteSql(CustomTableDefinitionModel tableModel, Long id) {
if (id == null) {
// 确保 ID 不为空,这是删除操作的必要条件
log.error("Delete error: rowId required. Table: {}.{}",
tableModel.getDorisDatabase(), tableModel.getDorisTable());
// 使用与更新逻辑一致的错误码
throw ComposeException.build(BizExceptionCodeEnum.fieldRequiredButEmpty, "行ID");
}
return "DELETE FROM `" + BuildSqlUtil.escapeSqlString(tableModel.getDorisDatabase()) + "`.`"
+ BuildSqlUtil.escapeSqlString(tableModel.getDorisTable()) + "` WHERE id = " + id;
}
}

View File

@@ -0,0 +1,202 @@
package com.xspaceagi.compose.domain.util.dml;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Map;
import java.util.stream.Collectors;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.domain.util.BuildSqlUtil;
import com.xspaceagi.compose.domain.util.SnowflakeIdWorker;
import com.xspaceagi.compose.sdk.enums.DefaultTableFieldEnum;
import com.xspaceagi.compose.sdk.enums.TableFieldTypeEnum;
import com.xspaceagi.system.spec.exception.BizExceptionCodeEnum;
import com.xspaceagi.system.spec.exception.ComposeException;
import lombok.extern.slf4j.Slf4j;
/**
* 插入数据SQL语句工具类
*/
@Slf4j
public final class InsertTableDMLUtil {
private InsertTableDMLUtil() {
}
/**
* 构建插入数据的SQL语句 (MySQL)
*/
public static String buildMysqlInsertSql(CustomTableDefinitionModel tableModel, Map<String, Object> rowData) {
// MySQL不生成id字段
return buildInsertSqlInternal(tableModel, rowData, false, true);
}
/**
* 构建插入数据的SQL语句 (Doris主键id需主动生成)
*/
public static String buildDorisInsertSql(CustomTableDefinitionModel tableModel, Map<String, Object> rowData) {
// Doris强制生成id字段
return buildInsertSqlInternal(tableModel, rowData, true, false);
}
/**
* 公共方法,生成 insert SQL
*
* @param tableModel 表结构
* @param rowData 行数据
* @param forceId 是否强制生成主键idDoris用
* @param skipId 是否跳过主键idMySQL用
*/
private static String buildInsertSqlInternal(CustomTableDefinitionModel tableModel, Map<String, Object> rowData,
boolean forceId, boolean skipId) {
if (rowData == null || rowData.isEmpty()) {
throw ComposeException.build(BizExceptionCodeEnum.composeInsertDataEmpty);
}
String idField = DefaultTableFieldEnum.ID.getFieldName();
// Doris 需要主动生成主键id
if (forceId) {
String id = SnowflakeIdWorker.nextIdStr();
rowData.put(idField, id);
}
// 固定插入创建时间和修改时间字段
String createdField = DefaultTableFieldEnum.CREATED.getFieldName();
String modifiedField = DefaultTableFieldEnum.MODIFIED.getFieldName();
// 获取东八区当前系统时间
ZoneId zoneId = ZoneId.of("Asia/Shanghai");
String currentTime = OffsetDateTime.now(zoneId).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
// 覆写创建时间和修改时间
rowData.put(createdField, currentTime);
rowData.put(modifiedField, currentTime);
// 构建字段类型Map
Map<String, Integer> fieldTypeMap = null;
Map<String, String> fieldDefaultMap = null;
if (tableModel.getFieldList() != null) {
fieldTypeMap = tableModel.getFieldList().stream()
.filter(f -> f != null && f.getFieldName() != null && f.getFieldType() != null)
.collect(Collectors.toMap(CustomFieldDefinitionModel::getFieldName,
CustomFieldDefinitionModel::getFieldType, (a, b) -> a));
fieldDefaultMap = tableModel.getFieldList().stream()
.filter(f -> f != null && f.getFieldName() != null && f.getDefaultValue() != null)
.collect(Collectors.toMap(CustomFieldDefinitionModel::getFieldName,
CustomFieldDefinitionModel::getDefaultValue, (a, b) -> a));
}
StringBuilder columns = new StringBuilder();
StringBuilder values = new StringBuilder();
for (Map.Entry<String, Object> entry : rowData.entrySet()) {
String fieldName = entry.getKey();
// MySQL 跳过 id 字段
if (skipId && idField.equalsIgnoreCase(fieldName)) {
continue;
}
Object value = entry.getValue();
columns.append("`").append(BuildSqlUtil.escapeSqlString(fieldName)).append("`, ");
Integer fieldType = fieldTypeMap != null ? fieldTypeMap.get(fieldName) : null;
String defaultVal = fieldDefaultMap != null ? fieldDefaultMap.get(fieldName) : null;
boolean isValueEmpty = (value == null) || (value instanceof String && ((String) value).trim().isEmpty());
// 布尔类型
if (fieldType != null && TableFieldTypeEnum.BOOLEAN.getCode().equals(fieldType)) {
String boolStr = isValueEmpty ? defaultVal : String.valueOf(value);
if ("true".equalsIgnoreCase(boolStr) || "1".equals(boolStr)) {
values.append("1, ");
} else if ("false".equalsIgnoreCase(boolStr) || "0".equals(boolStr)) {
values.append("0, ");
} else if (isValueEmpty) {
// 空值时设置为NULL
values.append("NULL, ");
} else {
// 非空值但无法识别的布尔值,抛出异常
log.error("Boolean conversion failed, fieldName: {}, value: {}", fieldName, value);
String errorMessage = String.format("布尔类型转换失败,字段名: %s, 字段值: %s, 支持的值: true/false/1/0",
fieldName, value);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
continue;
}
// 日期类型
if (fieldType != null && TableFieldTypeEnum.DATE.getCode().equals(fieldType)) {
if (!isValueEmpty && value instanceof String) {
String dateStr = (String) value;
try {
OffsetDateTime odt = OffsetDateTime.parse(dateStr, DateTimeFormatter.ISO_OFFSET_DATE_TIME);
LocalDateTime ldt = odt.atZoneSameInstant(java.time.ZoneId.of("Asia/Shanghai"))
.toLocalDateTime();
String mysqlDateStr = ldt.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
values.append("'").append(mysqlDateStr).append("', ");
continue;
} catch (Exception e) {
log.error("Date conversion failed, fieldName: {}, value: {}", fieldName, value, e);
String errorMessage = String.format("日期类型转换失败,字段名: %s, 字段值: %s, 错误: %s",
fieldName, value, e.getMessage());
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
}
}
// 其它类型
if (isValueEmpty) {
if (defaultVal != null && !defaultVal.isEmpty()) {
if (fieldType != null && (TableFieldTypeEnum.INTEGER.getCode().equals(fieldType)
|| TableFieldTypeEnum.NUMBER.getCode().equals(fieldType))) {
// 验证默认值是否为有效数字
try {
Double.parseDouble(defaultVal);
values.append(defaultVal).append(", ");
} catch (NumberFormatException e) {
log.error("Numeric default conversion failed, fieldName: {}, defaultValue: {}", fieldName, defaultVal, e);
String errorMessage = String.format("数值类型默认值转换失败,字段名: %s, 默认值: %s, 错误: %s",
fieldName, defaultVal, e.getMessage());
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
} else {
values.append("'").append(BuildSqlUtil.escapeSqlString(defaultVal)).append("', ");
}
} else {
values.append("NULL, ");
}
} else if (value instanceof Number) {
values.append(value).append(", ");
} else {
// 对于字符串类型的数值字段,验证是否为有效数字
if (fieldType != null && (TableFieldTypeEnum.INTEGER.getCode().equals(fieldType)
|| TableFieldTypeEnum.NUMBER.getCode().equals(fieldType))) {
try {
Double.parseDouble(value.toString());
values.append(value).append(", ");
} catch (NumberFormatException e) {
log.error("Numeric conversion failed, fieldName: {}, value: {}", fieldName, value, e);
String errorMessage = String.format("数值类型转换失败,字段名: %s, 字段值: %s, 错误: %s",
fieldName, value, e.getMessage());
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
} else {
values.append("'").append(BuildSqlUtil.escapeSqlString(value == null ? "" : value.toString()))
.append("', ");
}
}
}
// 去掉最后的逗号和空格
columns.setLength(columns.length() - 2);
values.setLength(values.length() - 2);
var sql = "INSERT INTO `" + BuildSqlUtil.escapeSqlString(tableModel.getDorisDatabase()) + "`.`"
+ BuildSqlUtil.escapeSqlString(tableModel.getDorisTable()) + "` ("
+ columns + ") VALUES (" + values + ")";
log.debug("Built INSERT INTO SQL for {}.{}: \n{}", tableModel.getDorisDatabase(), tableModel.getDorisTable(),
sql);
return sql;
}
}

View File

@@ -0,0 +1,120 @@
package com.xspaceagi.compose.domain.util.dml;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.LocalDateTime;
import java.util.Map;
import java.util.stream.Collectors;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.util.BuildSqlUtil;
import com.xspaceagi.compose.sdk.enums.TableFieldTypeEnum;
import com.xspaceagi.system.spec.exception.BizExceptionCodeEnum;
import com.xspaceagi.system.spec.exception.ComposeException;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public final class UpdateTableDMLUtil {
private UpdateTableDMLUtil() {}
/**
* 构建更新数据的SQL语句,根据id来修改行数
*/
public static String buildUpdateSql(CustomTableDefinitionModel tableModel, Map<String, Object> rowData, Long id) {
if (id == null) {
throw ComposeException.build(BizExceptionCodeEnum.fieldRequiredButEmpty, "行ID");
}
if (rowData == null || rowData.isEmpty()) {
throw ComposeException.build(BizExceptionCodeEnum.composeUpdateDataEmpty);
}
// 构建字段类型Map
Map<String, Integer> fieldTypeMap = null;
if (tableModel.getFieldList() != null) {
fieldTypeMap = tableModel.getFieldList().stream()
.filter(f -> f != null && f.getFieldName() != null)
.collect(Collectors.toMap(CustomFieldDefinitionModel::getFieldName,
CustomFieldDefinitionModel::getFieldType));
}
StringBuilder setClause = new StringBuilder();
for (Map.Entry<String, Object> entry : rowData.entrySet()) {
String fieldName = entry.getKey();
Object value = entry.getValue();
Integer fieldType = fieldTypeMap != null ? fieldTypeMap.get(fieldName) : null;
boolean isValueEmpty = (value == null) || (value instanceof String && ((String) value).trim().isEmpty());
setClause.append("`").append(BuildSqlUtil.escapeSqlString(fieldName)).append("` = ");
// 优先处理布尔类型
if (fieldType != null && TableFieldTypeEnum.BOOLEAN.getCode().equals(fieldType)) {
String boolStr = isValueEmpty ? null : String.valueOf(value);
if ("true".equalsIgnoreCase(boolStr) || "1".equals(boolStr)) {
setClause.append("1");
} else if ("false".equalsIgnoreCase(boolStr) || "0".equals(boolStr)) {
setClause.append("0");
} else if (isValueEmpty) {
// 空值时设置为NULL
setClause.append("NULL");
} else {
// 非空值但无法识别的布尔值,抛出异常
log.error("Boolean conversion failed, fieldName: {}, value: {}", fieldName, value);
String errorMessage = String.format("布尔类型转换失败,字段名: %s, 字段值: %s, 支持的值: true/false/1/0",
fieldName, value);
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
} else if (fieldType != null && TableFieldTypeEnum.DATE.getCode().equals(fieldType)) {
// 新增:处理日期类型,自动转换 ISO 8601 到 MySQL DATETIME东八区
if (!isValueEmpty && value instanceof String) {
String dateStr = (String) value;
try {
OffsetDateTime odt = OffsetDateTime.parse(dateStr, DateTimeFormatter.ISO_OFFSET_DATE_TIME);
LocalDateTime ldt = odt.atZoneSameInstant(java.time.ZoneId.of("Asia/Shanghai")).toLocalDateTime();
String mysqlDateStr = ldt.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
setClause.append("'").append(mysqlDateStr).append("'");
} catch (Exception e) {
log.error("Date conversion failed, fieldName: {}, value: {}", fieldName, value, e);
String errorMessage = String.format("日期类型转换失败,字段名: %s, 字段值: %s, 错误: %s",
fieldName, value, e.getMessage());
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
} else if (isValueEmpty) {
setClause.append("NULL");
} else {
setClause.append("'").append(BuildSqlUtil.escapeSqlString(value == null ? "" : value.toString())).append("'");
}
} else if (isValueEmpty) {
setClause.append("NULL");
} else if (value instanceof Number) {
setClause.append(value);
} else {
// 对于字符串类型的数值字段,验证是否为有效数字
if (fieldType != null && (TableFieldTypeEnum.INTEGER.getCode().equals(fieldType)
|| TableFieldTypeEnum.NUMBER.getCode().equals(fieldType))) {
try {
Double.parseDouble(value.toString());
setClause.append(value);
} catch (NumberFormatException e) {
log.error("Numeric conversion failed, fieldName: {}, value: {}", fieldName, value, e);
String errorMessage = String.format("数值类型转换失败,字段名: %s, 字段值: %s, 错误: %s",
fieldName, value, e.getMessage());
throw ComposeException.build(BizExceptionCodeEnum.composeSqlExecuteFailed, errorMessage);
}
} else {
setClause.append("'").append(BuildSqlUtil.escapeSqlString(value == null ? "" : value.toString())).append("'");
}
}
setClause.append(", ");
}
// 去掉最后的逗号和空格
if (setClause.length() > 2) {
setClause.setLength(setClause.length() - 2);
}
return "UPDATE `" + BuildSqlUtil.escapeSqlString(tableModel.getDorisDatabase()) + "`.`"
+ BuildSqlUtil.escapeSqlString(tableModel.getDorisTable()) + "` SET "
+ setClause + " WHERE id = " + id;
}
}

View File

@@ -0,0 +1,188 @@
package com.xspaceagi.compose.domain.util.excel;
import java.io.InputStream;
import java.util.*;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import cn.idev.excel.FastExcel;
import cn.idev.excel.context.AnalysisContext;
import cn.idev.excel.event.AnalysisEventListener;
import lombok.extern.slf4j.Slf4j;
/**
* excel工具类,使用的 fastExcel 库
* <p>
* 文档: https://idev.cn/fastexcel/zh-CN/docs/advance_api
* </p>
*/
@Slf4j
public class ExcelUtils {
// 匹配 "中文名称(english_name)" 格式的正则表达式
private static final Pattern HEADER_PATTERN = Pattern.compile("([^(]+)\\(([^)]+)\\)");
/**
* 动态读取excel表头和数据无Java Bean绑定兼容无数据时获取表头
*
* @param inputStream excel输入流
* @return ReadExcelDataVoheader为表头data为数据
*/
public static ReadExcelDataVo readExcel(InputStream inputStream, CustomTableDefinitionModel tableDefinitionModel) {
final List<Map<String, Object>> dataList = new ArrayList<>();
final List<String> originalHeader = new ArrayList<>();
final List<CustomFieldDefinitionModel> matchedHeaderFields = new ArrayList<>();
final List<String> newFieldsToCreate = new ArrayList<>();
final Map<Integer, String> headerIndexToFinalKeyMap = new LinkedHashMap<>();
// 获取表定义字段列表
final List<CustomFieldDefinitionModel> fieldList = tableDefinitionModel.getFieldList();
// 创建字段名 -> 字段定义 的映射
final Map<String, CustomFieldDefinitionModel> fieldNameMap = fieldList.stream()
.collect(Collectors.toMap(CustomFieldDefinitionModel::getFieldName, Function.identity(), (a, b) -> a));
// 创建字段描述 -> 字段定义 的映射
final Map<String, CustomFieldDefinitionModel> fieldDescriptionMap = fieldList.stream()
.filter(f -> StringUtils.isNotBlank(f.getFieldDescription()))
.collect(Collectors.toMap(CustomFieldDefinitionModel::getFieldDescription, Function.identity(),
(a, b) -> a));
FastExcel.read(inputStream, new AnalysisEventListener<Map<Integer, String>>() {
@Override
public void invoke(Map<Integer, String> data, AnalysisContext context) {
// 数据行
Map<String, Object> row = new LinkedHashMap<>();
headerIndexToFinalKeyMap.forEach((index, key) -> {
row.put(key, data.get(index));
});
dataList.add(row);
}
@Override
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
//去掉 两边的空格
var trimmedValues = headMap.values().stream()
.filter(StringUtils::isNotBlank)
.map(ExcelUtils::cleanWhitespace)
.toList();
originalHeader.addAll(trimmedValues);
for (int i = 0; i < headMap.size(); i++) {
String rawHeader = Optional.ofNullable(headMap.get(i))
.map(ExcelUtils::cleanWhitespace)
.orElse(null);
if (StringUtils.isBlank(rawHeader)) {
log.warn("Excel header empty at index {}, skip column", i);
continue;
}
rawHeader = ExcelUtils.cleanWhitespace(rawHeader);
Matcher matcher = HEADER_PATTERN.matcher(rawHeader);
String parsedDesc = null;
String parsedName = null;
if (matcher.matches()) {
// 匹配 "中文(英文)" 格式
parsedDesc = ExcelUtils.cleanWhitespace(matcher.group(1));
parsedName = ExcelUtils.cleanWhitespace(matcher.group(2));
}
CustomFieldDefinitionModel matchedField = null;
// 策略1: 根据解析出的英文名匹配 fieldName
if (parsedName != null) {
matchedField = fieldNameMap.get(parsedName);
}
// 策略2: 如果策略1失败则根据解析出的中文名匹配 fieldDescription
if (matchedField == null && parsedDesc != null) {
matchedField = fieldDescriptionMap.get(parsedDesc);
}
// 策略3: 如果策略1和2都失败或非 "中文(英文)" 格式),则使用原始表头尝试匹配
if (matchedField == null) {
// 3.1 尝试匹配英文名 fieldName
matchedField = fieldNameMap.get(rawHeader);
// 3.2 尝试匹配中文名 fieldDescription
if (matchedField == null) {
matchedField = fieldDescriptionMap.get(rawHeader);
}
}
if (matchedField != null) {
// 找到了匹配的字段
matchedHeaderFields.add(matchedField);
headerIndexToFinalKeyMap.put(i, matchedField.getFieldName());
} else {
// 未找到匹配字段,将其视为一个要创建的新字段
log.info("No match for header '{}', create new field.", rawHeader);
newFieldsToCreate.add(rawHeader);
// 使用原始表头作为临时key以便在数据行中引用
headerIndexToFinalKeyMap.put(i, rawHeader);
}
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
log.info("Excel read done. Matched {} fields, will create {} new.", matchedHeaderFields.size(),
newFieldsToCreate.size());
}
}).headRowNumber(1).sheet().doRead();
ReadExcelDataVo result = new ReadExcelDataVo();
result.setOriginalHeader(originalHeader);
result.setHeader(matchedHeaderFields);
result.setData(dataList);
result.setNewFieldsToCreate(newFieldsToCreate);
return result;
}
/**
* 根据新字段的映射关系重映射Excel数据中的key
*/
public static void remapExcelDataKeys(List<Map<String, Object>> excelData, Map<String, String> newFieldMapping) {
if (CollectionUtils.isEmpty(excelData) || CollectionUtils.isEmpty(newFieldMapping)) {
return;
}
for (Map<String, Object> row : excelData) {
Map<String, Object> updatedRow = new java.util.LinkedHashMap<>();
for (Map.Entry<String, Object> entry : row.entrySet()) {
String oldKey = entry.getKey();
Object value = entry.getValue();
String newKey = newFieldMapping.getOrDefault(oldKey, oldKey);
updatedRow.put(newKey, value);
}
row.clear();
row.putAll(updatedRow);
}
}
/**
* 去掉空格,以及特殊的空格,比如: <0xa0>(即 Unicode U+00A0No-Break Space的字符
* 优化版本:保留换行符等格式字符,只处理空格类字符
* @param str
* @return
*/
public static String cleanWhitespace(String str) {
if (str == null) {
return "";
}
// 替换特殊空格字符
String cleaned = str.replace("\u00A0", " ");
// 只合并连续的空格字符(不包括换行符),保留换行符等格式
cleaned = cleaned.replaceAll("[ \\f\\v]+", " ").trim();
return cleaned;
}
}

View File

@@ -0,0 +1,34 @@
package com.xspaceagi.compose.domain.util.excel;
import java.util.List;
import java.util.Map;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import lombok.Getter;
import lombok.Setter;
/**
* 读取excel数据返回结果
*/
@Getter
@Setter
public class ReadExcelDataVo {
/**
* 原始标题行
*/
private List<String> originalHeader;
/**
* 处理后的标题行
*/
private List<CustomFieldDefinitionModel> header;
/**
* 数据行
*/
private List<Map<String, Object>> data;
/**
* 需要新增的字段,表头是纯中文的
*/
private List<String> newFieldsToCreate;
}

View File

@@ -0,0 +1,238 @@
package com.xspaceagi.compose.domain.util;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.fail;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.List;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.sdk.enums.TableFieldTypeEnum;
import com.xspaceagi.system.spec.enums.YnEnum;
import lombok.extern.slf4j.Slf4j;
/**
* Integration tests for {@link DorisTableDdlUtil} that execute DDL against a
* real Doris database.
* NOTE: Requires a running Doris instance accessible at DORIS_JDBC_URL.
*/
@Slf4j
@TestMethodOrder(MethodOrderer.OrderAnnotation.class) // Ensure setup/teardown order
class DorisTableDdlUtilIT {
// Database connection details (consider using a properties file or test
// profile)
// 从 application-dev.yml 获取
private static final String DORIS_JDBC_URL_BASE = "jdbc:mysql://127.0.0.1:9030/?serverTimezone=Asia/Shanghai&characterEncoding=utf-8&allowMultiQueries=true&socketTimeout=300000&useSSL=false";
private static final String DORIS_JDBC_URL_TEST_DB = "jdbc:mysql://127.0.0.1:9030/test_db?serverTimezone=Asia/Shanghai&characterEncoding=utf-8&allowMultiQueries=true&socketTimeout=300000&useSSL=false";
private static final String DORIS_USER = "admin";
private static final String DORIS_PASSWORD = ""; // 你的密码
private static final String TEST_DB_NAME = "test_db";
private static final String TEST_TABLE_NAME = "test_table";
// 单节点开发环境下副本数应该设置为1
private static final int REPLICATION_NUM = 1;
private static Connection baseConnection; // Connection without specific DB
private static Connection testDbConnection; // Connection to test_db
private CustomTableDefinitionModel tableModel;
@BeforeAll
static void setupDatabase() throws SQLException {
log.debug("Setting up database for integration tests...");
try {
// Load the driver (optional for modern JDBC, but good practice)
Class.forName("com.mysql.cj.jdbc.Driver");
baseConnection = DriverManager.getConnection(DORIS_JDBC_URL_BASE, DORIS_USER, DORIS_PASSWORD);
log.debug("Base connection established.");
// Create test database if it doesn't exist
try (Statement stmt = baseConnection.createStatement()) {
log.debug("Creating database if not exists: " + TEST_DB_NAME);
stmt.execute("CREATE DATABASE IF NOT EXISTS " + TEST_DB_NAME);
log.debug("Database check/creation complete.");
}
// Establish connection to the specific test database
testDbConnection = DriverManager.getConnection(DORIS_JDBC_URL_TEST_DB, DORIS_USER, DORIS_PASSWORD);
log.debug("Connection to " + TEST_DB_NAME + " established.");
} catch (ClassNotFoundException e) {
log.error("MySQL JDBC Driver not found. Make sure it's in the classpath (scope=test).");
throw new RuntimeException(e);
} catch (SQLException e) {
log.error("Database connection failed: " + e.getMessage());
log.error("Check if Doris is running at 127.0.0.1:9030 and credentials are correct.");
throw e; // Re-throw to fail the setup
}
log.debug("Database setup complete.");
}
@AfterAll
static void tearDownDatabase() throws SQLException {
log.debug("Tearing down database...");
// Close connections first
if (testDbConnection != null && !testDbConnection.isClosed()) {
testDbConnection.close();
log.debug("Test DB connection closed.");
}
// Drop the test database carefully
if (baseConnection != null) {
try (Statement stmt = baseConnection.createStatement()) {
log.debug("Dropping database: " + TEST_DB_NAME);
// WARNING: Dropping the database can affect other tests or applications.
// Consider only dropping the table in @AfterEach or manual cleanup.
// stmt.execute("DROP DATABASE IF EXISTS " + TEST_DB_NAME);
log.debug(
"Database cleanup skipped (manual cleanup recommended or drop table in @AfterEach/BeforeEach).");
} finally {
if (!baseConnection.isClosed()) {
baseConnection.close();
log.debug("Base connection closed.");
}
}
}
log.debug("Database teardown complete.");
}
@BeforeEach
void setUpTable() throws SQLException {
// Drop the table before each test to ensure isolation
if (testDbConnection == null || testDbConnection.isClosed()) {
fail("Test DB connection is not available in @BeforeEach");
}
try (Statement stmt = testDbConnection.createStatement()) {
log.debug("Dropping table before test: " + TEST_DB_NAME + "." + TEST_TABLE_NAME);
stmt.execute("DROP TABLE IF EXISTS `" + TEST_DB_NAME + "`.`" + TEST_TABLE_NAME + "`");
log.debug("Table dropped.");
} catch (SQLException e) {
log.error("Failed to drop table " + TEST_DB_NAME + "." + TEST_TABLE_NAME + ": " + e.getMessage());
throw e;
}
// Setup the table model for the test
tableModel = new CustomTableDefinitionModel();
tableModel.setDorisDatabase(TEST_DB_NAME);
tableModel.setDorisTable(TEST_TABLE_NAME);
tableModel.setTableDescription("Integration Test Table 'Desc'");
}
// Helper method to create a field (same as in unit test)
private CustomFieldDefinitionModel createField(String name, TableFieldTypeEnum type, String description,
Integer nullableFlag, String defaultValue) {
CustomFieldDefinitionModel field = CustomFieldDefinitionModel.builder()
.fieldName(name)
.fieldType(type.getCode())
.fieldDescription(description)
.nullableFlag(nullableFlag)
.defaultValue(defaultValue)
.enabledFlag(YnEnum.Y.getKey())
.build();
return field;
}
// --- Integration Tests ---
@Test
@Order(1) // Execute CREATE first
@DisplayName("Execute CREATE TABLE SQL for basic table")
void testExecuteCreateTableSql_Basic() throws SQLException {
List<CustomFieldDefinitionModel> fields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "Primary Key", -1, null),
createField("name", TableFieldTypeEnum.STRING, "User Name", 1, "default_user"),
createField("score", TableFieldTypeEnum.NUMBER, "User Score", 1, "99.5"),
createField("is_active", TableFieldTypeEnum.BOOLEAN, "Activity Status", -1, "true"),
createField("created_at", TableFieldTypeEnum.DATE, "Creation timestamp", -1, null));
String actualSql = DorisTableDdlUtil.buildCreateTableSql(tableModel, fields);
log.debug("Executing CREATE SQL:\n" + actualSql);
assertDoesNotThrow(() -> {
if (testDbConnection == null || testDbConnection.isClosed()) {
fail("Test DB connection is not available for CREATE execution");
}
try (Statement stmt = testDbConnection.createStatement()) {
stmt.execute(actualSql);
log.debug("CREATE TABLE executed successfully.");
}
}, "CREATE TABLE statement failed to execute");
// Optional: Add verification by describing the table
// assertTableStructure(...)
}
@Test
@Order(2) // Execute ALTER after CREATE
@DisplayName("Execute ALTER TABLE SQL for adding, modifying, and dropping columns")
void testExecuteAlterTableSqls() throws SQLException {
if (testDbConnection == null || testDbConnection.isClosed()) {
fail("Test DB connection is not available for ALTER execution");
}
// 1. Create initial table state (required for ALTER)
List<CustomFieldDefinitionModel> initialFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "Old PK Comment", -1, null),
createField("to_drop", TableFieldTypeEnum.STRING, "Drop me", 1, null),
createField("to_keep", TableFieldTypeEnum.BOOLEAN, "Keep me", 1, "true"));
String createSql = DorisTableDdlUtil.buildCreateTableSql(tableModel, initialFields);
log.debug("Executing prerequisite CREATE SQL:\n" + createSql);
assertDoesNotThrow(() -> {
try (Statement stmt = testDbConnection.createStatement()) {
stmt.execute(createSql);
log.debug("Prerequisite CREATE TABLE executed successfully.");
}
}, "Prerequisite CREATE TABLE failed");
// 2. Define the new desired state
List<CustomFieldDefinitionModel> newFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "New PK Comment", -1, null), // Modify comment
createField("to_keep", TableFieldTypeEnum.BOOLEAN, "Keep me", 1, "true"), // Keep
createField("added_col", TableFieldTypeEnum.DATE, "Added Date", 1, null) // Add
);
// 3. Generate ALTER statements
List<String> alterSqls = DorisTableDdlUtil.buildAlterTableSqls(
tableModel.getDorisDatabase(),
tableModel.getDorisTable(),
initialFields, // Old state
newFields // New state
);
assertFalse(alterSqls.isEmpty(), "Expected ALTER statements to be generated");
log.debug("Generated ALTER SQLs: " + alterSqls);
// 4. Execute ALTER statements
assertDoesNotThrow(() -> {
try (Statement stmt = testDbConnection.createStatement()) {
for (String sql : alterSqls) {
log.debug("Executing ALTER SQL: " + sql);
// Note: Doris often prefers multiple changes in one ALTER statement,
// but executing separately is simpler for testing generated individual
// statements.
stmt.execute(sql);
log.debug("ALTER statement executed successfully: " + sql);
}
}
}, "One or more ALTER TABLE statements failed to execute");
// Optional: Verify final structure
// assertTableStructureAfterAlter(...)
}
// TODO: Add more integration test cases for specific scenarios if needed
}

View File

@@ -0,0 +1,272 @@
package com.xspaceagi.compose.domain.util;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import com.xspaceagi.compose.domain.model.CustomFieldDefinitionModel;
import com.xspaceagi.compose.domain.model.CustomTableDefinitionModel;
import com.xspaceagi.compose.sdk.enums.TableFieldTypeEnum;
import com.xspaceagi.system.spec.enums.YnEnum;
import com.xspaceagi.system.spec.exception.ComposeException;
import lombok.extern.slf4j.Slf4j;
/**
* Unit tests for {@link DorisTableDdlUtil}
*/
@Slf4j
class DorisTableDdlUtilTest {
private CustomTableDefinitionModel tableModel;
@BeforeEach
void setUp() {
tableModel = new CustomTableDefinitionModel();
tableModel.setDorisDatabase("test_db");
tableModel.setDorisTable("test_table");
tableModel.setTableDescription("Test Table Description with 'quotes'");
}
// Helper method to create a field
private CustomFieldDefinitionModel createField(String name, TableFieldTypeEnum type, String description,
Integer nullableFlag, String defaultValue) {
CustomFieldDefinitionModel field = CustomFieldDefinitionModel.builder()
.fieldName(name)
.fieldType(type.getCode())
.fieldDescription(description)
.nullableFlag(nullableFlag)
.defaultValue(defaultValue)
.enabledFlag(YnEnum.Y.getKey())
.build();
return field;
}
// --- Tests for buildCreateTableSql ---
@Test
@DisplayName("Build CREATE TABLE SQL for basic table")
void testBuildCreateTableSql_Basic() {
List<CustomFieldDefinitionModel> fields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "Primary Key", -1, null),
createField("name", TableFieldTypeEnum.STRING, "User Name", 1, "default_user"), // Corrected default
// value format for test
createField("created_at", TableFieldTypeEnum.DATE, "Creation timestamp", -1, null));
String expectedSql = "CREATE TABLE IF NOT EXISTS `test_db`.`test_table` (\n" +
" `id` INT NOT NULL COMMENT 'Primary Key',\n" +
" `name` VARCHAR(255) NULL DEFAULT 'default_user' COMMENT 'User Name',\n" +
" `created_at` DATETIME NOT NULL COMMENT 'Creation timestamp'\n" +
") \n" +
"ENGINE=OLAP\n" +
"DUPLICATE KEY(`id`)\n" +
"COMMENT 'Test Table Description with \\\'quotes\\\''\n" +
"DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" +
"PROPERTIES (\n" +
" \"replication_num\" = \"3\"\n" +
")";
String actualSql = DorisTableDdlUtil.buildCreateTableSql(tableModel, fields);
// assertEquals normalizes line endings, trim() handles leading/trailing
// whitespace
assertEquals(expectedSql.trim(), actualSql.trim());
}
@Test
@DisplayName("Build CREATE TABLE SQL with various default values")
void testBuildCreateTableSql_DefaultValues() {
List<CustomFieldDefinitionModel> fields = Arrays.asList(
createField("pk", TableFieldTypeEnum.INTEGER, "PK", -1, null),
createField("str_col", TableFieldTypeEnum.STRING, "String with default", 1, "hello world"),
createField("int_col", TableFieldTypeEnum.INTEGER, "Int with default", 1, "123"),
createField("num_col", TableFieldTypeEnum.NUMBER, "Decimal with default", 1, "99.99"),
createField("bool_col_true", TableFieldTypeEnum.BOOLEAN, "Boolean true default", 1, "true"),
createField("bool_col_false", TableFieldTypeEnum.BOOLEAN, "Boolean false default", 1, "0"),
createField("null_col", TableFieldTypeEnum.STRING, "String with NULL default", 1, "NULL"));
String expectedSql = "CREATE TABLE IF NOT EXISTS `test_db`.`test_table` (\n" +
" `pk` INT NOT NULL COMMENT 'PK',\n" +
" `str_col` VARCHAR(255) NULL DEFAULT 'hello world' COMMENT 'String with default',\n" +
" `int_col` INT NULL DEFAULT 123 COMMENT 'Int with default',\n" +
" `num_col` DECIMAL(20,6) NULL DEFAULT 99.99 COMMENT 'Decimal with default',\n" +
" `bool_col_true` TINYINT(1) NULL DEFAULT 1 COMMENT 'Boolean true default',\n" +
" `bool_col_false` TINYINT(1) NULL DEFAULT 0 COMMENT 'Boolean false default',\n" +
" `null_col` VARCHAR(255) NULL DEFAULT NULL COMMENT 'String with NULL default'\n" +
") \n" +
"ENGINE=OLAP\n" +
"DUPLICATE KEY(`pk`)\n" +
"COMMENT 'Test Table Description with \\\'quotes\\\''\n" +
"DISTRIBUTED BY HASH(`pk`) BUCKETS 10\n" +
"PROPERTIES (\n" +
" \"replication_num\" = \"3\"\n" +
")";
String actualSql = DorisTableDdlUtil.buildCreateTableSql(tableModel, fields);
assertEquals(expectedSql.trim(), actualSql.trim());
}
@Test
@DisplayName("Build CREATE TABLE SQL should throw exception for empty fields")
void testBuildCreateTableSql_EmptyFields() {
List<CustomFieldDefinitionModel> fields = Collections.emptyList();
String expectedMessage = "数据不存在";
ComposeException exception = assertThrows(ComposeException.class, () -> {
DorisTableDdlUtil.buildCreateTableSql(tableModel, fields);
});
assertEquals(expectedMessage, exception.getMessage());
}
@Test
@DisplayName("Build CREATE TABLE SQL should throw exception for null table name")
void testBuildCreateTableSql_NullTableName() {
tableModel.setDorisTable(null);
List<CustomFieldDefinitionModel> fields = Collections.singletonList(
createField("id", TableFieldTypeEnum.INTEGER, "ID", -1, null));
String expectedMessage = "数据不存在";
ComposeException exception = assertThrows(ComposeException.class, () -> {
DorisTableDdlUtil.buildCreateTableSql(tableModel, fields);
});
assertEquals(expectedMessage, exception.getMessage());
}
// --- Tests for buildAlterTableSqls ---
@Test
@DisplayName("Build ALTER TABLE SQL for adding a new column")
void testBuildAlterTableSqls_AddColumn() {
List<CustomFieldDefinitionModel> oldFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "PK", -1, null));
List<CustomFieldDefinitionModel> newFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "PK", -1, null),
createField("new_col", TableFieldTypeEnum.STRING, "Newly added column", 1, "new_val") // Corrected
// default
);
List<String> expectedSqls = Collections.singletonList(
"ALTER TABLE `test_db`.`test_table` ADD COLUMN `new_col` VARCHAR(255) NULL DEFAULT 'new_val' COMMENT 'Newly added column'");
List<String> actualSqls = DorisTableDdlUtil.buildAlterTableSqls("test_db", "test_table", oldFields, newFields);
assertEquals(expectedSqls, actualSqls);
}
@Test
@DisplayName("Build ALTER TABLE SQL for modifying a column comment")
void testBuildAlterTableSqls_ModifyComment() {
List<CustomFieldDefinitionModel> oldFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "Old Comment", -1, null));
List<CustomFieldDefinitionModel> newFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "New Comment 'with quote'", -1, null) // Changed comment
);
List<String> expectedSqls = Collections.singletonList(
// Note: MODIFY requires the full definition
"ALTER TABLE `test_db`.`test_table` MODIFY COLUMN `id` INT NOT NULL COMMENT 'New Comment \\\'with quote\\\''");
List<String> actualSqls = DorisTableDdlUtil.buildAlterTableSqls("test_db", "test_table", oldFields, newFields);
assertEquals(expectedSqls, actualSqls);
}
@Test
@DisplayName("Build ALTER TABLE SQL for dropping a column")
void testBuildAlterTableSqls_DropColumn() {
List<CustomFieldDefinitionModel> oldFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "PK", -1, null),
createField("to_drop", TableFieldTypeEnum.STRING, "This will be dropped", 1, null));
List<CustomFieldDefinitionModel> newFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "PK", -1, null));
List<String> expectedSqls = Collections.singletonList(
"ALTER TABLE `test_db`.`test_table` DROP COLUMN `to_drop`");
List<String> actualSqls = DorisTableDdlUtil.buildAlterTableSqls("test_db", "test_table", oldFields, newFields);
assertEquals(expectedSqls, actualSqls);
}
@Test
@DisplayName("Build ALTER TABLE SQL for adding, modifying comment, and dropping")
void testBuildAlterTableSqls_AddModifyDrop() {
List<CustomFieldDefinitionModel> oldFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "Old PK Comment", -1, null),
createField("to_drop", TableFieldTypeEnum.STRING, "Drop me", 1, null),
createField("to_keep", TableFieldTypeEnum.BOOLEAN, "Keep me", 1, "true"));
List<CustomFieldDefinitionModel> newFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "New PK Comment", -1, null), // Modify comment
createField("to_keep", TableFieldTypeEnum.BOOLEAN, "Keep me", 1, "true"), // Keep
createField("added_col", TableFieldTypeEnum.DATE, "Added Date", 1, null) // Add
);
List<String> actualSqls = DorisTableDdlUtil.buildAlterTableSqls("test_db", "test_table", oldFields, newFields);
// Order might vary depending on implementation details (add/modify vs drop
// loops)
// Check contents regardless of order
assertEquals(3, actualSqls.size());
assertTrue(actualSqls.contains(
"ALTER TABLE `test_db`.`test_table` ADD COLUMN `added_col` DATETIME NULL COMMENT 'Added Date'"));
assertTrue(actualSqls.contains(
"ALTER TABLE `test_db`.`test_table` MODIFY COLUMN `id` INT NOT NULL COMMENT 'New PK Comment'"));
assertTrue(actualSqls.contains("ALTER TABLE `test_db`.`test_table` DROP COLUMN `to_drop`"));
}
@Test
@DisplayName("Build ALTER TABLE SQL should return empty list for no changes")
void testBuildAlterTableSqls_NoChanges() {
List<CustomFieldDefinitionModel> oldFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "Comment", -1, null));
List<CustomFieldDefinitionModel> newFields = Arrays.asList(
createField("id", TableFieldTypeEnum.INTEGER, "Comment", -1, null) // Identical
);
List<String> actualSqls = DorisTableDdlUtil.buildAlterTableSqls("test_db", "test_table", oldFields, newFields);
assertTrue(actualSqls.isEmpty());
}
@Test
@DisplayName("Build ALTER TABLE SQL should handle null/empty lists")
void testBuildAlterTableSqls_NullEmptyLists() {
List<CustomFieldDefinitionModel> fields = Collections.singletonList(
createField("id", TableFieldTypeEnum.INTEGER, "ID", -1, null));
// Old null
List<String> addSqls = DorisTableDdlUtil.buildAlterTableSqls("test_db", "test_table", null, fields);
assertEquals(1, addSqls.size());
assertTrue(addSqls.get(0).contains("ADD COLUMN `id`"));
// New null
List<String> dropSqls = DorisTableDdlUtil.buildAlterTableSqls("test_db", "test_table", fields, null);
assertEquals(1, dropSqls.size());
assertTrue(dropSqls.get(0).contains("DROP COLUMN `id`"));
// Both null
List<String> noChangeSqls = DorisTableDdlUtil.buildAlterTableSqls("test_db", "test_table", null, null);
assertTrue(noChangeSqls.isEmpty());
// Both empty
List<String> noChangeSqlsEmpty = DorisTableDdlUtil.buildAlterTableSqls("test_db", "test_table",
Collections.emptyList(), Collections.emptyList());
assertTrue(noChangeSqlsEmpty.isEmpty());
}
@Test
@DisplayName("Build ALTER TABLE SQL should not generate change for only type/nullability diff (currently)")
void testBuildAlterTableSqls_TypeOrNullabilityChange_NoOp() {
List<CustomFieldDefinitionModel> oldFields = Arrays.asList(
createField("col1", TableFieldTypeEnum.INTEGER, "Comment", -1, null) // INT, NOT NULL
);
List<CustomFieldDefinitionModel> newFields = Arrays.asList(
createField("col1", TableFieldTypeEnum.STRING, "Comment", 1, null) // VARCHAR, NULL
);
List<String> actualSqls = DorisTableDdlUtil.buildAlterTableSqls("test_db", "test_table", oldFields, newFields);
// Currently, only comment changes are detected for existing columns
assertTrue(actualSqls.isEmpty(), "Expected no ALTER statements as only type/nullability changed");
}
}