diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SchemaReader.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SchemaReader.java deleted file mode 100644 index 2654bfe..0000000 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SchemaReader.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (C) 2026 github.com/MaloneTalk - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * limitations under the License. - */ -package io.github.malonetalk.agent.datasource; - -import io.github.malonetalk.entity.Datasource; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -@Slf4j -@Component -@AllArgsConstructor -public class SchemaReader { - - private final DynamicDataSourceManager dynamicDataSourceManager; - - public List getTableSchema(Datasource datasource, String tableName) { - javax.sql.DataSource ds = dynamicDataSourceManager.getOrCreateDataSource(datasource); - - try (Connection conn = ds.getConnection()) { - Set primaryKeys = getPrimaryKeys(conn, tableName); - return getColumns(conn, tableName, primaryKeys); - } catch (SQLException e) { - log.error("Failed to read schema for table {}: {}", tableName, e.getMessage(), e); - throw new SchemaReadException( - "Failed to read schema for table " + tableName + ": " + e.getMessage(), e); - } - } - - private Set getPrimaryKeys(Connection conn, String tableName) throws SQLException { - Set pkColumns = new HashSet<>(); - DatabaseMetaData metaData = conn.getMetaData(); - - try (ResultSet rs = - metaData.getPrimaryKeys(conn.getCatalog(), conn.getSchema(), tableName)) { - while (rs.next()) { - pkColumns.add(rs.getString("COLUMN_NAME")); - } - } - - return pkColumns; - } - - private List getColumns(Connection conn, String tableName, Set primaryKeys) - throws SQLException { - List columns = new ArrayList<>(); - DatabaseMetaData metaData = conn.getMetaData(); - - try (ResultSet rs = - metaData.getColumns(conn.getCatalog(), conn.getSchema(), tableName, null)) { - while (rs.next()) { - String columnName = rs.getString("COLUMN_NAME"); - String typeName = rs.getString("TYPE_NAME"); - int columnSize = rs.getInt("COLUMN_SIZE"); - String nullableStr = rs.getString("IS_NULLABLE"); - boolean nullable = "YES".equalsIgnoreCase(nullableStr); - String defaultValue = rs.getString("COLUMN_DEF"); - String remarks = rs.getString("REMARKS"); - boolean isPk = primaryKeys.contains(columnName); - - columns.add( - new ColumnInfo( - columnName, - typeName, - columnSize, - nullable, - defaultValue, - isPk, - remarks)); - } - } - - return columns; - } - - public static class SchemaReadException extends RuntimeException { - public SchemaReadException(String message, Throwable cause) { - super(message, cause); - } - } -} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/ExecuteSqlTool.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/ExecuteSqlTool.java index c7396bf..fcbabb0 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/ExecuteSqlTool.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/ExecuteSqlTool.java @@ -19,12 +19,12 @@ import io.agentscope.core.tool.Tool; import io.agentscope.core.tool.ToolParam; -import io.github.malonetalk.agent.datasource.QueryResult; -import io.github.malonetalk.agent.datasource.SqlExecutor; -import io.github.malonetalk.agent.datasource.SqlExecutor.SqlExecutionException; -import io.github.malonetalk.agent.datasource.SqlExecutor.SqlSecurityException; +import io.github.malonetalk.common.QueryResult; import io.github.malonetalk.entity.Datasource; import io.github.malonetalk.enums.Status; +import io.github.malonetalk.infrastructure.SqlExecutor; +import io.github.malonetalk.infrastructure.SqlExecutor.SqlExecutionException; +import io.github.malonetalk.infrastructure.SqlExecutor.SqlSecurityException; import io.github.malonetalk.service.DatasourceService; import java.util.List; import lombok.AllArgsConstructor; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetTableSchemaTool.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetTableSchemaTool.java index d7c7032..85f1580 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetTableSchemaTool.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetTableSchemaTool.java @@ -19,24 +19,20 @@ import io.agentscope.core.tool.Tool; import io.agentscope.core.tool.ToolParam; -import io.github.malonetalk.agent.datasource.ColumnInfo; -import io.github.malonetalk.agent.datasource.SchemaReader; -import io.github.malonetalk.agent.datasource.SchemaReader.SchemaReadException; -import io.github.malonetalk.entity.Datasource; -import io.github.malonetalk.enums.Status; -import io.github.malonetalk.service.DatasourceService; -import java.util.List; -import lombok.AllArgsConstructor; +import io.github.malonetalk.common.ToolResult; +import io.github.malonetalk.dto.PageResponse; +import io.github.malonetalk.dto.semantic.TableSchemaSemanticPrompt; +import io.github.malonetalk.service.semantic.SemanticMergeService; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @Slf4j @Component -@AllArgsConstructor +@RequiredArgsConstructor public class GetTableSchemaTool implements MarkAgentTool { - private final DatasourceService dataSourceService; - private final SchemaReader schemaReader; + private final SemanticMergeService semanticMergeService; @Tool( name = "get_table_schema", @@ -45,43 +41,30 @@ public class GetTableSchemaTool implements MarkAgentTool { + " type, whether it is primary key, whether it allows null, default value" + " and column comments. This tool should be called to understand the table" + " structure before generating SQL.") - public String getTableSchema( + public ToolResult getTableSchema( @ToolParam(name = "table_name", description = "The table name to query schema for") - String tableName) { - List activeDataSources = - dataSourceService.findByStatus(Status.ACTIVE.getCode()); - - if (activeDataSources.isEmpty()) { - return "No active datasource available, cannot get table schema."; - } - - if (activeDataSources.size() > 1) { - log.warn( - "Found {} active data sources, using the first one.", activeDataSources.size()); - } - - Datasource datasource = activeDataSources.get(0); - + String tableName, + @ToolParam( + name = "column_page", + description = "Optional column page number, default is 1") + Integer columnPage, + @ToolParam( + name = "column_page_size", + description = + "Optional column page size, default is 20, maximum is 100") + Integer columnPageSize) { try { - List columns = schemaReader.getTableSchema(datasource, tableName); - return formatSchema(tableName, columns); - } catch (SchemaReadException e) { - return "Failed to get table schema: " + e.getMessage(); + int resolvedPage = PageResponse.resolvePage(columnPage); + int resolvedPageSize = PageResponse.resolvePageSize(columnPageSize); + return ToolResult.success( + semanticMergeService.getTableSchema(tableName, resolvedPage, resolvedPageSize)); + } catch (IllegalStateException e) { + return ToolResult.error("Data source parsing failed", e.getMessage()); + } catch (IllegalArgumentException e) { + return ToolResult.error("Invalid arguments", e.getMessage()); + } catch (RuntimeException e) { + log.error("Failed to get schema for table {}: {}", tableName, e.getMessage(), e); + return ToolResult.error("Failed to retrieve schema", e.getMessage()); } } - - private String formatSchema(String tableName, List columns) { - if (columns.isEmpty()) { - return "Table " + tableName + " does not exist or has no column information."; - } - - StringBuilder sb = new StringBuilder(); - sb.append("Schema of table ").append(tableName).append(":\n"); - - for (ColumnInfo col : columns) { - sb.append(" - ").append(col.toString()).append("\n"); - } - - return sb.toString(); - } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetTablesTool.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetTablesTool.java index c98eb91..dba4b7a 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetTablesTool.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/GetTablesTool.java @@ -18,42 +18,42 @@ package io.github.malonetalk.agent.tools; import io.agentscope.core.tool.Tool; -import io.github.malonetalk.entity.Datasource; -import io.github.malonetalk.entity.TableInfo; -import io.github.malonetalk.enums.Status; -import io.github.malonetalk.service.DatasourceService; -import io.github.malonetalk.service.semantic.table.TableSemanticService; -import java.util.Collections; -import java.util.List; -import lombok.AllArgsConstructor; +import io.agentscope.core.tool.ToolParam; +import io.github.malonetalk.agent.tools.response.TablePromptResponse; +import io.github.malonetalk.common.ToolResult; +import io.github.malonetalk.dto.PageResponse; +import io.github.malonetalk.service.semantic.SemanticMergeService; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @Slf4j @Component -@AllArgsConstructor +@RequiredArgsConstructor public class GetTablesTool implements MarkAgentTool { - private final DatasourceService dataSourceService; - private final TableSemanticService tableSemanticService; + private final SemanticMergeService semanticMergeService; @Tool(name = "get_tables", description = "获取数据库中的表信息，包括表名和表描述") - public List getTables() { - List activeDataSources = - dataSourceService.findByStatus(Status.ACTIVE.getCode()); - - if (activeDataSources.isEmpty()) { - return Collections.emptyList(); - } - - if (activeDataSources.size() > 1) { - log.warn( - "Found {} active data sources, using the first one. This may cause data" - + " inconsistency.", - activeDataSources.size()); + public ToolResult> getTables( + @ToolParam(name = "page", description = "Optional page number, default is 1") + Integer page, + @ToolParam( + name = "page_size", + description = "Optional page size, default is 20, maximum is 100") + Integer pageSize) { + try { + int resolvedPage = PageResponse.resolvePage(page); + int resolvedPageSize = PageResponse.resolvePageSize(pageSize); + return ToolResult.success( + semanticMergeService.getVisibleTablePromptPage(resolvedPage, resolvedPageSize)); + } catch (IllegalStateException e) { + return ToolResult.error("Data source parsing failed", e.getMessage()); + } catch (IllegalArgumentException e) { + return ToolResult.error("Invalid arguments", e.getMessage()); + } catch (RuntimeException e) { + log.error("Failed to retrieve visible tables: {}", e.getMessage(), e); + return ToolResult.error("Failed to retrieve tables", e.getMessage()); } - - Datasource dataSource = activeDataSources.get(0); - return tableSemanticService.listTableInfosByDatasourceId(dataSource.getId()); } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/response/ColumnPromptResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/response/ColumnPromptResponse.java new file mode 100644 index 0000000..42dcd86 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/response/ColumnPromptResponse.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.agent.tools.response; + +public record ColumnPromptResponse( + String name, + String type, + Boolean primaryKey, + Boolean nullable, + String defaultValue, + String description) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/response/TablePromptResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/response/TablePromptResponse.java new file mode 100644 index 0000000..863317c --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/response/TablePromptResponse.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.agent.tools.response; + +import java.util.List; + +public record TablePromptResponse( + String name, String domain, String description, List relations) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/response/TableRelationResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/response/TableRelationResponse.java new file mode 100644 index 0000000..1da72c7 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/agent/tools/response/TableRelationResponse.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.agent.tools.response; + +import java.util.List; + +public record TableRelationResponse( + String relationType, + String source, + String sourceTableName, + List sourceColumnNames, + String targetTableName, + List targetColumnNames, + String description) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/QueryResult.java b/data-agent-backend/src/main/java/io/github/malonetalk/common/QueryResult.java similarity index 98% rename from data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/QueryResult.java rename to data-agent-backend/src/main/java/io/github/malonetalk/common/QueryResult.java index e63d13b..520396d 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/QueryResult.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/common/QueryResult.java @@ -15,7 +15,7 @@ * along with this program. If not, see . * limitations under the License. */ -package io.github.malonetalk.agent.datasource; +package io.github.malonetalk.common; import java.util.ArrayList; import java.util.LinkedHashMap; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/common/ToolError.java b/data-agent-backend/src/main/java/io/github/malonetalk/common/ToolError.java new file mode 100644 index 0000000..8514999 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/common/ToolError.java @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.common; + +public record ToolError(String code, String message) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/common/ToolResult.java b/data-agent-backend/src/main/java/io/github/malonetalk/common/ToolResult.java new file mode 100644 index 0000000..91edc3b --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/common/ToolResult.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.common; + +public record ToolResult(boolean success, T data, ToolError error) { + + public static ToolResult success(T data) { + return new ToolResult<>(true, data, null); + } + + public static ToolResult error(String code, String message) { + return new ToolResult<>(false, null, new ToolError(code, message)); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableColumnSemanticController.java b/data-agent-backend/src/main/java/io/github/malonetalk/controller/ColumnSemanticController.java similarity index 97% rename from data-agent-backend/src/main/java/io/github/malonetalk/controller/TableColumnSemanticController.java rename to data-agent-backend/src/main/java/io/github/malonetalk/controller/ColumnSemanticController.java index d6bb3e3..4b8e1db 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableColumnSemanticController.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/controller/ColumnSemanticController.java @@ -18,7 +18,7 @@ package io.github.malonetalk.controller; import io.github.malonetalk.common.Result; -import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.PageResponse; import io.github.malonetalk.dto.semantic.BatchResetColumnSemanticRequest; import io.github.malonetalk.dto.semantic.ColumnSemanticPageQuery; import io.github.malonetalk.dto.semantic.ColumnSemanticResponse; @@ -41,7 +41,7 @@ @RestController @RequestMapping("/api/semantic/tables/columns/{tableName}") @RequiredArgsConstructor -public class TableColumnSemanticController { +public class ColumnSemanticController { private final ColumnSemanticService columnSemanticService; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableRelationSemanticController.java b/data-agent-backend/src/main/java/io/github/malonetalk/controller/RelationSemanticController.java similarity index 74% rename from data-agent-backend/src/main/java/io/github/malonetalk/controller/TableRelationSemanticController.java rename to data-agent-backend/src/main/java/io/github/malonetalk/controller/RelationSemanticController.java index bd3ffab..2273ed4 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableRelationSemanticController.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/controller/RelationSemanticController.java @@ -18,10 +18,12 @@ package io.github.malonetalk.controller; import io.github.malonetalk.common.Result; -import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.PageResponse; import io.github.malonetalk.dto.semantic.BatchDeleteLogicalTableRelationRequest; import io.github.malonetalk.dto.semantic.BindLogicalTableRelationRequest; import io.github.malonetalk.dto.semantic.LogicalTableRelationResponse; +import io.github.malonetalk.dto.semantic.RelationCandidateColumnResponse; +import io.github.malonetalk.dto.semantic.RelationCandidateTableResponse; import io.github.malonetalk.dto.semantic.RelationSemanticPageQuery; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationEnabledRequest; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationRequest; @@ -42,13 +44,13 @@ import org.springframework.web.bind.annotation.RestController; @RestController -@RequestMapping("/api/semantic/tables/relations/{tableName}") +@RequestMapping("/api/semantic/tables") @RequiredArgsConstructor -public class TableRelationSemanticController { +public class RelationSemanticController { private final RelationSemanticService relationSemanticService; - @GetMapping + @GetMapping("/{tableName}/relations") public Result> listByTable( @PathVariable @NotBlank String tableName, @Valid RelationSemanticPageQuery query) { return Result.success( @@ -63,21 +65,21 @@ public Result> listByTable( query.sortOrder()))); } - @PostMapping + @PostMapping("/{tableName}/relations") public Result create( @PathVariable @NotBlank String tableName, @Valid @RequestBody BindLogicalTableRelationRequest request) { return Result.success(relationSemanticService.createRelationSemantic(tableName, request)); } - @PutMapping + @PutMapping("/{tableName}/relations") public Result update( @PathVariable @NotBlank String tableName, @Valid @RequestBody UpdateLogicalTableRelationRequest request) { return Result.success(relationSemanticService.updateRelationSemantic(tableName, request)); } - @PutMapping("/enabled") + @PutMapping("/{tableName}/relations/enabled") public Result updateEnabled( @PathVariable @NotBlank String tableName, @Valid @RequestBody UpdateLogicalTableRelationEnabledRequest request) { @@ -85,7 +87,7 @@ public Result updateEnabled( relationSemanticService.updateRelationSemanticEnabled(tableName, request)); } - @DeleteMapping("/{relationId}") + @DeleteMapping("/{tableName}/relations/{relationId}") public Result delete( @PathVariable @NotBlank String tableName, @PathVariable @NotNull @Min(1) Integer relationId, @@ -95,7 +97,7 @@ public Result delete( datasourceId, tableName, relationId)); } - @DeleteMapping("/batch") + @DeleteMapping("/{tableName}/relations/batch") public Result deleteBatch( @PathVariable @NotBlank String tableName, @Valid @RequestBody BatchDeleteLogicalTableRelationRequest request) { @@ -103,4 +105,25 @@ public Result deleteBatch( relationSemanticService.deleteRelationSemantics( request.datasourceId(), tableName, request.relationIds())); } + + @GetMapping("/relations/candidate/tables") + public Result> candidateTables( + @Valid RelationSemanticPageQuery query) { + return Result.success(relationSemanticService.getCandidateTablePage(query)); + } + + @GetMapping("/{tableName}/relations/candidate/columns") + public Result> candidateColumns( + @PathVariable @NotBlank String tableName, @Valid RelationSemanticPageQuery query) { + return Result.success( + relationSemanticService.getCandidateColumnPage( + new RelationSemanticPageQuery( + query.datasourceId(), + tableName, + query.page(), + query.pageSize(), + query.keyword(), + query.enabled(), + query.sortOrder()))); + } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableSemanticController.java b/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableSemanticController.java index 14dcb26..a2e1e53 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableSemanticController.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/controller/TableSemanticController.java @@ -18,7 +18,7 @@ package io.github.malonetalk.controller; import io.github.malonetalk.common.Result; -import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.PageResponse; import io.github.malonetalk.dto.semantic.BatchResetTableSemanticRequest; import io.github.malonetalk.dto.semantic.TableSemanticPageQuery; import io.github.malonetalk.dto.semantic.TableSemanticResponse; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/convertor/ColumnSemanticConverter.java b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/ColumnSemanticConverter.java new file mode 100644 index 0000000..3bd182b --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/ColumnSemanticConverter.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.convertor; + +import io.github.malonetalk.dto.semantic.ColumnSemanticResponse; +import io.github.malonetalk.entity.ColumnSemantic; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; + +@Mapper(componentModel = "spring") +public interface ColumnSemanticConverter { + + @Mapping(target = "physicalColumnDescription", expression = "java(null)") + @Mapping(target = "typeName", expression = "java(null)") + @Mapping(target = "primaryKey", expression = "java(null)") + @Mapping(target = "hasPhysicalColumn", constant = "true") + @Mapping( + target = "effective", + expression = "java(Boolean.TRUE.equals(columnSemantic.getIsVisible()))") + @Mapping(target = "invalidReason", expression = "java(null)") + ColumnSemanticResponse toResponse(ColumnSemantic columnSemantic); +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/convertor/LogicalTableRelationConverter.java b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/LogicalTableRelationConverter.java new file mode 100644 index 0000000..c690c8d --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/LogicalTableRelationConverter.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.convertor; + +import io.github.malonetalk.dto.semantic.LogicalTableRelationResponse; +import io.github.malonetalk.entity.LogicalTableRelation; +import io.github.malonetalk.service.semantic.relation.LogicalTableRelationHelper; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class LogicalTableRelationConverter { + + private final LogicalTableRelationHelper logicalTableRelationHelper; + + public LogicalTableRelationResponse toResponse(LogicalTableRelation relation) { + List sourceColumns = + logicalTableRelationHelper.fromJson( + relation.getSourceColumnNamesJson(), "sourceColumnNames"); + List targetColumns = + logicalTableRelationHelper.fromJson( + relation.getTargetColumnNamesJson(), "targetColumnNames"); + return new LogicalTableRelationResponse( + relation.getId(), + logicalTableRelationHelper.buildRelationKey( + relation.getSourceTableName(), + sourceColumns, + relation.getTargetTableName(), + targetColumns), + relation.getDatasourceId(), + LogicalTableRelationHelper.RELATION_SOURCE_LOGICAL, + relation.getSourceTableName(), + sourceColumns, + relation.getTargetTableName(), + targetColumns, + relation.getRelationType(), + relation.getDescription(), + relation.getIsEnabled(), + Boolean.TRUE.equals(relation.getIsEnabled()), + null, + relation.getCreateTime(), + relation.getUpdateTime()); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/convertor/TableSemanticConverter.java b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/TableSemanticConverter.java new file mode 100644 index 0000000..aa8c58d --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/convertor/TableSemanticConverter.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.convertor; + +import io.github.malonetalk.dto.semantic.TableSemanticResponse; +import io.github.malonetalk.entity.TableSemantic; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; + +@Mapper(componentModel = "spring") +public interface TableSemanticConverter { + + @Mapping(target = "physicalTableDescription", expression = "java(null)") + @Mapping(target = "hasPhysicalTable", constant = "true") + @Mapping( + target = "effective", + expression = "java(Boolean.TRUE.equals(tableSemantic.getIsVisible()))") + @Mapping(target = "invalidReason", expression = "java(null)") + TableSemanticResponse toResponse(TableSemantic tableSemantic); +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/pagination/PageResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/PageResponse.java similarity index 98% rename from data-agent-backend/src/main/java/io/github/malonetalk/dto/pagination/PageResponse.java rename to data-agent-backend/src/main/java/io/github/malonetalk/dto/PageResponse.java index e8ff6f8..113304b 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/dto/pagination/PageResponse.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/PageResponse.java @@ -15,7 +15,7 @@ * along with this program. If not, see . * limitations under the License. */ -package io.github.malonetalk.dto.pagination; +package io.github.malonetalk.dto; import java.util.Collections; import java.util.List; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationCandidateColumnResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationCandidateColumnResponse.java new file mode 100644 index 0000000..a201b1e --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationCandidateColumnResponse.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +public record RelationCandidateColumnResponse( + String columnName, String description, String typeName, Boolean primaryKey) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationCandidateTableResponse.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationCandidateTableResponse.java new file mode 100644 index 0000000..56f355f --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/RelationCandidateTableResponse.java @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +public record RelationCandidateTableResponse(String tableName, String domain, String description) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/TableSchemaSemanticPrompt.java b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/TableSchemaSemanticPrompt.java new file mode 100644 index 0000000..cd4ff63 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/dto/semantic/TableSchemaSemanticPrompt.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.dto.semantic; + +import io.github.malonetalk.agent.tools.response.ColumnPromptResponse; +import io.github.malonetalk.dto.PageResponse; + +public record TableSchemaSemanticPrompt( + String name, + String domain, + String description, + PageResponse columns) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/ColumnInfo.java b/data-agent-backend/src/main/java/io/github/malonetalk/entity/Column.java similarity index 95% rename from data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/ColumnInfo.java rename to data-agent-backend/src/main/java/io/github/malonetalk/entity/Column.java index 142a8c0..20cd617 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/ColumnInfo.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/entity/Column.java @@ -15,9 +15,9 @@ * along with this program. If not, see . * limitations under the License. */ -package io.github.malonetalk.agent.datasource; +package io.github.malonetalk.entity; -public record ColumnInfo( +public record Column( String columnName, String typeName, int columnSize, diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnInfo.java b/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnSemantic.java similarity index 97% rename from data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnInfo.java rename to data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnSemantic.java index 45bb532..811028f 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnInfo.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/entity/ColumnSemantic.java @@ -21,7 +21,7 @@ import lombok.Data; @Data -public class ColumnInfo { +public class ColumnSemantic { private Integer id; private Integer datasourceId; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/entity/PhysicalTable.java b/data-agent-backend/src/main/java/io/github/malonetalk/entity/PhysicalTable.java new file mode 100644 index 0000000..60e4846 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/entity/PhysicalTable.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.entity; + +import lombok.Data; + +@Data +public class PhysicalTable { + + private String tableName; + private String tableDescription; + private Integer datasourceId; +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableRelationInfo.java b/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableRelationInfo.java new file mode 100644 index 0000000..9e1a90f --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableRelationInfo.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.entity; + +import java.util.List; + +public record TableRelationInfo( + String sourceTableName, + List sourceColumnNames, + String targetTableName, + List targetColumnNames, + String relationType, + String description) {} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableInfo.java b/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableSemantic.java similarity index 97% rename from data-agent-backend/src/main/java/io/github/malonetalk/entity/TableInfo.java rename to data-agent-backend/src/main/java/io/github/malonetalk/entity/TableSemantic.java index cb2c16f..6a094de 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableInfo.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/entity/TableSemantic.java @@ -21,7 +21,7 @@ import lombok.Data; @Data -public class TableInfo { +public class TableSemantic { private Integer id; private String tableName; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/DataSourceType.java b/data-agent-backend/src/main/java/io/github/malonetalk/enums/DataSourceType.java similarity index 97% rename from data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/DataSourceType.java rename to data-agent-backend/src/main/java/io/github/malonetalk/enums/DataSourceType.java index 0e48e00..e599f42 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/DataSourceType.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/enums/DataSourceType.java @@ -15,7 +15,7 @@ * along with this program. If not, see . * limitations under the License. */ -package io.github.malonetalk.agent.datasource; +package io.github.malonetalk.enums; import java.util.Arrays; import java.util.Optional; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/exception/GlobalExceptionHandler.java b/data-agent-backend/src/main/java/io/github/malonetalk/exception/GlobalExceptionHandler.java index df0c536..e0cc323 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/exception/GlobalExceptionHandler.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/exception/GlobalExceptionHandler.java @@ -39,4 +39,11 @@ public Result handleBadRequest(Exception exception) { String message = exception.getMessage() == null ? "Bad request" : exception.getMessage(); return Result.error(400, message); } + + @ExceptionHandler(SemanticSchemaException.class) + public Result handleSemanticSchema(SemanticSchemaException exception) { + String message = + exception.getMessage() == null ? "Semantic schema error" : exception.getMessage(); + return Result.error(400, message); + } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/exception/SemanticSchemaException.java b/data-agent-backend/src/main/java/io/github/malonetalk/exception/SemanticSchemaException.java new file mode 100644 index 0000000..67c6cad --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/exception/SemanticSchemaException.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.exception; + +public class SemanticSchemaException extends RuntimeException { + + public SemanticSchemaException(String message) { + super(message); + } + + public SemanticSchemaException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/DynamicDataSourceManager.java b/data-agent-backend/src/main/java/io/github/malonetalk/infrastructure/DynamicDataSourceManager.java similarity index 97% rename from data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/DynamicDataSourceManager.java rename to data-agent-backend/src/main/java/io/github/malonetalk/infrastructure/DynamicDataSourceManager.java index 137150c..10579b2 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/DynamicDataSourceManager.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/infrastructure/DynamicDataSourceManager.java @@ -15,11 +15,12 @@ * along with this program. If not, see . * limitations under the License. */ -package io.github.malonetalk.agent.datasource; +package io.github.malonetalk.infrastructure; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import io.github.malonetalk.entity.Datasource; +import io.github.malonetalk.enums.DataSourceType; import jakarta.annotation.PreDestroy; import java.util.concurrent.ConcurrentHashMap; import javax.sql.DataSource; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/infrastructure/SchemaReader.java b/data-agent-backend/src/main/java/io/github/malonetalk/infrastructure/SchemaReader.java new file mode 100644 index 0000000..aec6f19 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/infrastructure/SchemaReader.java @@ -0,0 +1,246 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.infrastructure; + +import io.github.malonetalk.entity.Column; +import io.github.malonetalk.entity.Datasource; +import io.github.malonetalk.entity.PhysicalTable; +import io.github.malonetalk.entity.TableRelationInfo; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +@AllArgsConstructor +public class SchemaReader { + + private final DynamicDataSourceManager dynamicDataSourceManager; + + public List getTableSchema(Datasource datasource, String tableName) { + javax.sql.DataSource ds = dynamicDataSourceManager.getOrCreateDataSource(datasource); + + try (Connection conn = ds.getConnection()) { + Set primaryKeys = getPrimaryKeys(conn, tableName); + return getColumns(conn, tableName, primaryKeys); + } catch (SQLException e) { + log.error("Failed to read schema for table {}: {}", tableName, e.getMessage(), e); + throw new SchemaReadException( + "Failed to read schema for table " + tableName + ": " + e.getMessage(), e); + } + } + + public List getTables(Datasource datasource) { + javax.sql.DataSource ds = dynamicDataSourceManager.getOrCreateDataSource(datasource); + try (Connection conn = ds.getConnection()) { + DatabaseMetaData metaData = conn.getMetaData(); + List tables = new ArrayList<>(); + try (ResultSet rs = + metaData.getTables( + conn.getCatalog(), conn.getSchema(), null, new String[] {"TABLE"})) { + while (rs.next()) { + String tableName = rs.getString("TABLE_NAME"); + if (tableName == null || tableName.isBlank()) { + continue; + } + PhysicalTable tableInfo = new PhysicalTable(); + tableInfo.setTableName(tableName); + tableInfo.setTableDescription(normalizeBlankToNull(rs.getString("REMARKS"))); + tableInfo.setDatasourceId(datasource.getId()); + tables.add(tableInfo); + } + } + return tables; + } catch (SQLException e) { + log.error( + "Failed to read tables for datasource {}: {}", + datasource.getId(), + e.getMessage(), + e); + throw new SchemaReadException( + "Failed to read tables for datasource " + + datasource.getId() + + ": " + + e.getMessage(), + e); + } + } + + public List getImportedRelations(Datasource datasource, String tableName) { + javax.sql.DataSource ds = dynamicDataSourceManager.getOrCreateDataSource(datasource); + try (Connection conn = ds.getConnection()) { + DatabaseMetaData metaData = conn.getMetaData(); + Map relationMap = new LinkedHashMap<>(); + try (ResultSet rs = + metaData.getImportedKeys(conn.getCatalog(), conn.getSchema(), tableName)) { + while (rs.next()) { + String sourceTable = rs.getString("FKTABLE_NAME"); + String sourceColumn = rs.getString("FKCOLUMN_NAME"); + String targetTable = rs.getString("PKTABLE_NAME"); + String targetColumn = rs.getString("PKCOLUMN_NAME"); + if (anyBlank(sourceTable, sourceColumn, targetTable, targetColumn)) { + continue; + } + String key = + buildRelationKey( + sourceTable, + sourceColumn, + targetTable, + targetColumn, + normalizeBlankToNull(rs.getString("FK_NAME"))); + TableRelationInfo existing = relationMap.get(key); + if (existing != null) { + List mergedSource = + mergeList(existing.sourceColumnNames(), sourceColumn); + List mergedTarget = + mergeList(existing.targetColumnNames(), targetColumn); + relationMap.put( + key, + new TableRelationInfo( + sourceTable, + mergedSource, + targetTable, + mergedTarget, + "foreign_key", + null)); + } else { + relationMap.put( + key, + new TableRelationInfo( + sourceTable, + List.of(sourceColumn), + targetTable, + List.of(targetColumn), + "foreign_key", + null)); + } + } + } + return List.copyOf(relationMap.values()); + } catch (SQLException e) { + log.error( + "Failed to read imported relations for table {} in datasource {}: {}", + tableName, + datasource.getId(), + e.getMessage(), + e); + throw new SchemaReadException( + "Failed to read imported relations for table " + + tableName + + ": " + + e.getMessage(), + e); + } + } + + private Set getPrimaryKeys(Connection conn, String tableName) throws SQLException { + Set pkColumns = new HashSet<>(); + DatabaseMetaData metaData = conn.getMetaData(); + + try (ResultSet rs = + metaData.getPrimaryKeys(conn.getCatalog(), conn.getSchema(), tableName)) { + while (rs.next()) { + pkColumns.add(rs.getString("COLUMN_NAME")); + } + } + + return pkColumns; + } + + private List getColumns(Connection conn, String tableName, Set primaryKeys) + throws SQLException { + List columns = new ArrayList<>(); + DatabaseMetaData metaData = conn.getMetaData(); + + try (ResultSet rs = + metaData.getColumns(conn.getCatalog(), conn.getSchema(), tableName, null)) { + while (rs.next()) { + String columnName = rs.getString("COLUMN_NAME"); + String typeName = rs.getString("TYPE_NAME"); + int columnSize = rs.getInt("COLUMN_SIZE"); + String nullableStr = rs.getString("IS_NULLABLE"); + boolean nullable = "YES".equalsIgnoreCase(nullableStr); + String defaultValue = rs.getString("COLUMN_DEF"); + String remarks = rs.getString("REMARKS"); + boolean isPk = primaryKeys.contains(columnName); + + columns.add( + new Column( + columnName, + typeName, + columnSize, + nullable, + defaultValue, + isPk, + remarks)); + } + } + + return columns; + } + + private static String normalizeBlankToNull(String value) { + if (value == null || value.isBlank()) { + return null; + } + return value.trim(); + } + + private static boolean anyBlank(String... values) { + for (String value : values) { + if (value == null || value.isBlank()) { + return true; + } + } + return false; + } + + private static String buildRelationKey( + String sourceTable, + String sourceColumn, + String targetTable, + String targetColumn, + String foreignKeyName) { + if (foreignKeyName != null) { + return sourceTable + "|" + targetTable + "|" + foreignKeyName; + } + return sourceTable + "|" + sourceColumn + "|" + targetTable + "|" + targetColumn; + } + + private static List mergeList(List existing, String newValue) { + List merged = new ArrayList<>(existing); + merged.add(newValue); + return merged; + } + + public static class SchemaReadException extends RuntimeException { + public SchemaReadException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SqlExecutor.java b/data-agent-backend/src/main/java/io/github/malonetalk/infrastructure/SqlExecutor.java similarity index 97% rename from data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SqlExecutor.java rename to data-agent-backend/src/main/java/io/github/malonetalk/infrastructure/SqlExecutor.java index 855204e..89e762d 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/agent/datasource/SqlExecutor.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/infrastructure/SqlExecutor.java @@ -15,8 +15,9 @@ * along with this program. If not, see . * limitations under the License. */ -package io.github.malonetalk.agent.datasource; +package io.github.malonetalk.infrastructure; +import io.github.malonetalk.common.QueryResult; import io.github.malonetalk.entity.Datasource; import java.sql.Connection; import java.sql.PreparedStatement; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticMapper.java similarity index 70% rename from data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java rename to data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticMapper.java index 470e761..9e62522 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticInfoMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/ColumnSemanticMapper.java @@ -18,31 +18,35 @@ package io.github.malonetalk.mapper; import io.github.malonetalk.dto.semantic.ColumnSemanticPageQuery; -import io.github.malonetalk.entity.ColumnInfo; +import io.github.malonetalk.entity.ColumnSemantic; import java.util.List; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; @Mapper -public interface ColumnSemanticInfoMapper { +public interface ColumnSemanticMapper { - List selectByDatasourceId(@Param("datasourceId") Integer datasourceId); + List selectByDatasourceId(@Param("datasourceId") Integer datasourceId); - List selectByDatasourceIdAndTableName( + List selectByDatasourceIdAndTableName( @Param("datasourceId") Integer datasourceId, @Param("tableName") String tableName); - List selectPageByDatasourceIdAndTableName( + List selectPageByDatasourceIdAndTableName( @Param("query") ColumnSemanticPageQuery query, @Param("sortDescending") boolean sortDescending); - ColumnInfo selectByDatasourceIdAndTableNameAndColumnName( + List selectVisiblePageByDatasourceIdAndTableName( + @Param("query") ColumnSemanticPageQuery query, + @Param("sortDescending") boolean sortDescending); + + ColumnSemantic selectByDatasourceIdAndTableNameAndColumnName( @Param("datasourceId") Integer datasourceId, @Param("tableName") String tableName, @Param("columnName") String columnName); - int insert(ColumnInfo columnInfo); + int insert(ColumnSemantic columnSemantic); - int update(ColumnInfo columnInfo); + int update(ColumnSemantic columnSemantic); int deleteByDatasourceId(@Param("datasourceId") Integer datasourceId); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableSemanticMapper.java similarity index 70% rename from data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java rename to data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableSemanticMapper.java index 9c41458..7b32aea 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableInfoMapper.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/mapper/TableSemanticMapper.java @@ -18,27 +18,31 @@ package io.github.malonetalk.mapper; import io.github.malonetalk.dto.semantic.TableSemanticPageQuery; -import io.github.malonetalk.entity.TableInfo; +import io.github.malonetalk.entity.TableSemantic; import java.util.List; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; @Mapper -public interface TableInfoMapper { +public interface TableSemanticMapper { - int insert(TableInfo tableInfo); + int insert(TableSemantic tableSemantic); - int update(TableInfo tableInfo); + int update(TableSemantic tableSemantic); int deleteByDatasourceIdAndIds( @Param("datasourceId") Integer datasourceId, @Param("ids") List ids); - List selectByDatasourceId(@Param("datasourceId") Integer datasourceId); + List selectByDatasourceId(@Param("datasourceId") Integer datasourceId); - List selectPageByDatasourceId( + List selectPageByDatasourceId( @Param("query") TableSemanticPageQuery query, @Param("sortDescending") boolean sortDescending); - TableInfo selectByDatasourceIdAndTableName( + List selectVisiblePageByDatasourceId( + @Param("query") TableSemanticPageQuery query, + @Param("sortDescending") boolean sortDescending); + + TableSemantic selectByDatasourceIdAndTableName( @Param("datasourceId") Integer datasourceId, @Param("tableName") String tableName); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceService.java index 03e1b02..8fe3b6d 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceService.java @@ -37,4 +37,6 @@ public interface DatasourceService { List findByType(String type); boolean updateStatus(Integer id, String status); + + Datasource requireActiveDatasource(); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceServiceImpl.java index b3b6b87..ca75a63 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/DatasourceServiceImpl.java @@ -18,12 +18,15 @@ package io.github.malonetalk.service; import io.github.malonetalk.entity.Datasource; +import io.github.malonetalk.enums.Status; import io.github.malonetalk.mapper.DatasourceMapper; import java.time.LocalDateTime; import java.util.List; import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +@Slf4j @Service @AllArgsConstructor public class DatasourceServiceImpl implements DatasourceService { @@ -72,4 +75,20 @@ public List findByType(String type) { public boolean updateStatus(Integer id, String status) { return dataSourceMapper.updateStatus(id, status) > 0; } + + @Override + public Datasource requireActiveDatasource() { + List activeDataSources = + dataSourceMapper.selectByStatus(Status.ACTIVE.getCode()); + if (activeDataSources.isEmpty()) { + throw new IllegalStateException("没有活跃的数据源，请先激活一个数据源。"); + } + if (activeDataSources.size() > 1) { + List activeIds = activeDataSources.stream().map(Datasource::getId).toList(); + String message = "存在多个活跃数据源 " + activeIds + "，请仅保留一个活跃数据源。"; + log.error("发现 {} 个活跃数据源，拒绝隐式选择。activeIds={}", activeDataSources.size(), activeIds); + throw new IllegalStateException(message); + } + return activeDataSources.get(0); + } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java new file mode 100644 index 0000000..90080e0 --- /dev/null +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/SemanticMergeService.java @@ -0,0 +1,319 @@ +/* + * Copyright (C) 2026 github.com/MaloneTalk + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * limitations under the License. + */ +package io.github.malonetalk.service.semantic; + +import com.github.pagehelper.Page; +import com.github.pagehelper.PageHelper; +import io.github.malonetalk.agent.tools.response.ColumnPromptResponse; +import io.github.malonetalk.agent.tools.response.TablePromptResponse; +import io.github.malonetalk.agent.tools.response.TableRelationResponse; +import io.github.malonetalk.dto.PageResponse; +import io.github.malonetalk.dto.semantic.ColumnSemanticPageQuery; +import io.github.malonetalk.dto.semantic.TableSchemaSemanticPrompt; +import io.github.malonetalk.dto.semantic.TableSemanticPageQuery; +import io.github.malonetalk.entity.Column; +import io.github.malonetalk.entity.ColumnSemantic; +import io.github.malonetalk.entity.Datasource; +import io.github.malonetalk.entity.LogicalTableRelation; +import io.github.malonetalk.entity.TableRelationInfo; +import io.github.malonetalk.entity.TableSemantic; +import io.github.malonetalk.infrastructure.SchemaReader; +import io.github.malonetalk.mapper.ColumnSemanticMapper; +import io.github.malonetalk.mapper.LogicalTableRelationMapper; +import io.github.malonetalk.mapper.TableSemanticMapper; +import io.github.malonetalk.service.DatasourceService; +import io.github.malonetalk.service.semantic.relation.LogicalTableRelationHelper; +import io.github.malonetalk.utils.SemanticUtils; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class SemanticMergeService { + + private final DatasourceService datasourceService; + private final SchemaReader schemaReader; + private final TableSemanticMapper tableSemanticMapper; + private final ColumnSemanticMapper columnSemanticMapper; + private final LogicalTableRelationMapper logicalTableRelationMapper; + private final LogicalTableRelationHelper logicalTableRelationHelper; + + public PageResponse getVisibleTablePromptPage(int page, int pageSize) { + Datasource datasource = datasourceService.requireActiveDatasource(); + PageHelper.startPage(page, pageSize); + Page pageResult = + (Page) + tableSemanticMapper.selectVisiblePageByDatasourceId( + new TableSemanticPageQuery( + datasource.getId(), page, pageSize, null, "asc"), + false); + + Map> logicalRelationsBySource = + buildLogicalRelationsBySource(datasource.getId()); + + List items = + pageResult.stream() + .map( + t -> + new TablePromptResponse( + t.getTableName(), + SemanticUtils.normalizeBlankToNull(t.getDomain()), + SemanticUtils.normalizeBlankToNull( + t.getTableDescription()), + resolveVisibleRelations( + t.getTableName(), + logicalRelationsBySource, + datasource))) + .toList(); + + return PageResponse.of(items, pageResult.getTotal(), page, pageSize); + } + + public TableSchemaSemanticPrompt getTableSchema(String tableName, int page, int pageSize) { + Datasource datasource = datasourceService.requireActiveDatasource(); + String normalizedTableName = SemanticUtils.requireName(tableName, "tableName"); + + TableSemantic semanticTable = + tableSemanticMapper.selectByDatasourceIdAndTableName( + datasource.getId(), normalizedTableName); + if (semanticTable == null || !Boolean.TRUE.equals(semanticTable.getIsVisible())) { + throw new IllegalArgumentException("表 " + normalizedTableName + " 不存在或不可见。"); + } + + Map physicalByKey = new HashMap<>(); + try { + for (Column col : schemaReader.getTableSchema(datasource, normalizedTableName)) { + physicalByKey.put(col.columnName().toLowerCase(Locale.ROOT), col); + } + } catch (SchemaReader.SchemaReadException e) { + throw new IllegalArgumentException( + "无法读取表 " + normalizedTableName + " 的 Schema: " + e.getMessage(), e); + } + if (physicalByKey.isEmpty()) { + throw new IllegalArgumentException("表 " + normalizedTableName + " 不存在或没有可读列。"); + } + + PageHelper.startPage(page, pageSize); + Page pageResult = + (Page) + columnSemanticMapper.selectVisiblePageByDatasourceIdAndTableName( + new ColumnSemanticPageQuery( + datasource.getId(), + normalizedTableName, + page, + pageSize, + null, + "asc"), + false); + + List items = + pageResult.stream().map(c -> mapColumnPrompt(c, physicalByKey)).toList(); + + String description = + SemanticUtils.normalizeBlankToNull(semanticTable.getTableDescription()); + String domain = SemanticUtils.normalizeBlankToNull(semanticTable.getDomain()); + + PageResponse columnPage = + PageResponse.of(items, pageResult.getTotal(), page, pageSize); + return new TableSchemaSemanticPrompt(normalizedTableName, domain, description, columnPage); + } + + private List resolveVisibleRelations( + String sourceTableName, + Map> logicalRelationsBySource, + Datasource datasource) { + LinkedHashMap merged = new LinkedHashMap<>(); + + // 先添加物理 FK 关系 + List physicalRelations = + schemaReader.getImportedRelations(datasource, sourceTableName); + for (TableRelationInfo phys : physicalRelations) { + if (!areEndpointsVisible(datasource.getId(), phys)) { + continue; + } + String key = + buildRelationMergeKey( + phys.sourceTableName(), + phys.sourceColumnNames(), + phys.targetTableName(), + phys.targetColumnNames()); + merged.put( + key, + new TableRelationResponse( + phys.relationType(), + "physical", + phys.sourceTableName(), + phys.sourceColumnNames(), + phys.targetTableName(), + phys.targetColumnNames(), + phys.description())); + } + + // 逻辑关系覆盖物理关系 + List logicalRelations = + logicalRelationsBySource.getOrDefault( + sourceTableName.toLowerCase(Locale.ROOT), Collections.emptyList()); + for (LogicalTableRelation relation : logicalRelations) { + if (!Boolean.TRUE.equals(relation.getIsEnabled())) { + continue; + } + List sourceColumns; + List targetColumns; + try { + sourceColumns = + logicalTableRelationHelper.fromJson( + relation.getSourceColumnNamesJson(), "sourceColumnNames"); + targetColumns = + logicalTableRelationHelper.fromJson( + relation.getTargetColumnNamesJson(), "targetColumnNames"); + } catch (IllegalArgumentException e) { + log.warn("跳过无效的逻辑关系 id={}: {}", relation.getId(), e.getMessage()); + continue; + } + String key = + buildRelationMergeKey( + relation.getSourceTableName(), + sourceColumns, + relation.getTargetTableName(), + targetColumns); + merged.put( + key, + new TableRelationResponse( + relation.getRelationType(), + LogicalTableRelationHelper.RELATION_SOURCE_LOGICAL, + relation.getSourceTableName(), + sourceColumns, + relation.getTargetTableName(), + targetColumns, + relation.getDescription())); + } + + return List.copyOf(merged.values()); + } + + private boolean areEndpointsVisible(Integer datasourceId, TableRelationInfo relation) { + TableSemantic sourceTable = + tableSemanticMapper.selectByDatasourceIdAndTableName( + datasourceId, relation.sourceTableName()); + if (sourceTable != null && !Boolean.TRUE.equals(sourceTable.getIsVisible())) { + return false; + } + TableSemantic targetTable = + tableSemanticMapper.selectByDatasourceIdAndTableName( + datasourceId, relation.targetTableName()); + if (targetTable != null && !Boolean.TRUE.equals(targetTable.getIsVisible())) { + return false; + } + for (String colName : relation.sourceColumnNames()) { + ColumnSemantic col = + columnSemanticMapper.selectByDatasourceIdAndTableNameAndColumnName( + datasourceId, relation.sourceTableName(), colName); + if (col != null && !Boolean.TRUE.equals(col.getIsVisible())) { + return false; + } + } + for (String colName : relation.targetColumnNames()) { + ColumnSemantic col = + columnSemanticMapper.selectByDatasourceIdAndTableNameAndColumnName( + datasourceId, relation.targetTableName(), colName); + if (col != null && !Boolean.TRUE.equals(col.getIsVisible())) { + return false; + } + } + return true; + } + + private String buildRelationMergeKey( + String sourceTable, + List sourceColumns, + String targetTable, + List targetColumns) { + String sourceColSig = + sourceColumns.stream() + .map(c -> c.toLowerCase(Locale.ROOT)) + .sorted() + .reduce((a, b) -> a + "|" + b) + .orElse(""); + String targetColSig = + targetColumns.stream() + .map(c -> c.toLowerCase(Locale.ROOT)) + .sorted() + .reduce((a, b) -> a + "|" + b) + .orElse(""); + return sourceTable.toLowerCase(Locale.ROOT) + + "|" + + sourceColSig + + "|" + + targetTable.toLowerCase(Locale.ROOT) + + "|" + + targetColSig; + } + + private ColumnPromptResponse mapColumnPrompt( + ColumnSemantic semanticColumn, Map physicalByKey) { + Column physicalColumn = + physicalByKey.get(semanticColumn.getColumnName().toLowerCase(Locale.ROOT)); + String description = + SemanticUtils.normalizeBlankToNull(semanticColumn.getColumnDescription()); + if (description == null && physicalColumn != null) { + description = SemanticUtils.normalizeBlankToNull(physicalColumn.remarks()); + } + String typeText = ""; + boolean primaryKey = false; + boolean nullable = true; + String defaultValue = null; + if (physicalColumn != null) { + StringBuilder typeBuilder = new StringBuilder(physicalColumn.typeName()); + if (physicalColumn.columnSize() > 0) { + typeBuilder.append("(").append(physicalColumn.columnSize()).append(")"); + } + typeText = typeBuilder.toString(); + primaryKey = physicalColumn.primaryKey(); + nullable = physicalColumn.nullable(); + defaultValue = SemanticUtils.normalizeBlankToNull(physicalColumn.defaultValue()); + } + return new ColumnPromptResponse( + semanticColumn.getColumnName(), + typeText, + primaryKey, + nullable, + defaultValue, + description); + } + + private Map> buildLogicalRelationsBySource( + Integer datasourceId) { + List allRelations = + logicalTableRelationMapper.selectByDatasourceId(datasourceId); + Map> result = new HashMap<>(); + for (LogicalTableRelation relation : allRelations) { + String key = relation.getSourceTableName().toLowerCase(Locale.ROOT); + result.computeIfAbsent(key, k -> new ArrayList<>()).add(relation); + } + return result; + } +} diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticService.java index d014abe..cd08abd 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticService.java @@ -17,7 +17,7 @@ */ package io.github.malonetalk.service.semantic.column; -import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.PageResponse; import io.github.malonetalk.dto.semantic.ColumnSemanticPageQuery; import io.github.malonetalk.dto.semantic.ColumnSemanticResponse; import io.github.malonetalk.dto.semantic.ColumnSemanticUpdateRequest; diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java index 295f1e2..5e1eb12 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/column/ColumnSemanticServiceImpl.java @@ -20,12 +20,13 @@ import com.github.pagehelper.Page; import com.github.pagehelper.PageHelper; import io.github.malonetalk.common.SemanticConstants; -import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.convertor.ColumnSemanticConverter; +import io.github.malonetalk.dto.PageResponse; import io.github.malonetalk.dto.semantic.ColumnSemanticPageQuery; import io.github.malonetalk.dto.semantic.ColumnSemanticResponse; import io.github.malonetalk.dto.semantic.ColumnSemanticUpdateRequest; -import io.github.malonetalk.entity.ColumnInfo; -import io.github.malonetalk.mapper.ColumnSemanticInfoMapper; +import io.github.malonetalk.entity.ColumnSemantic; +import io.github.malonetalk.mapper.ColumnSemanticMapper; import io.github.malonetalk.service.DatasourceService; import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; @@ -41,7 +42,8 @@ public class ColumnSemanticServiceImpl implements ColumnSemanticService { private final DatasourceService datasourceService; - private final ColumnSemanticInfoMapper columnSemanticInfoMapper; + private final ColumnSemanticMapper columnSemanticMapper; + private final ColumnSemanticConverter columnSemanticConverter; @Override public PageResponse getColumnPage(ColumnSemanticPageQuery query) { @@ -56,9 +58,9 @@ public PageResponse getColumnPage(ColumnSemanticPageQuer boolean sortDescending = SemanticConstants.SORT_ORDER_DESC.equalsIgnoreCase(query.sortOrder()); PageHelper.startPage(pageNumber, pageSize); - Page page = - (Page) - columnSemanticInfoMapper.selectPageByDatasourceIdAndTableName( + Page page = + (Page) + columnSemanticMapper.selectPageByDatasourceIdAndTableName( new ColumnSemanticPageQuery( query.datasourceId(), normalizedTableName, @@ -67,7 +69,8 @@ public PageResponse getColumnPage(ColumnSemanticPageQuer SemanticUtils.normalizeBlankToNull(query.keyword()), query.sortOrder()), sortDescending); - List responses = page.stream().map(this::mapResponse).toList(); + List responses = + page.stream().map(columnSemanticConverter::toResponse).toList(); return PageResponse.of(responses, page.getTotal(), pageNumber, pageSize); } @@ -76,20 +79,20 @@ public void updateColumnSemantic(String tableName, ColumnSemanticUpdateRequest r requireDatasource(request.datasourceId()); String normalizedTableName = SemanticUtils.requireName(tableName, "tableName"); String normalizedColumnName = SemanticUtils.requireName(request.columnName(), "columnName"); - ColumnInfo existing = - columnSemanticInfoMapper.selectByDatasourceIdAndTableNameAndColumnName( + ColumnSemantic existing = + columnSemanticMapper.selectByDatasourceIdAndTableNameAndColumnName( request.datasourceId(), normalizedTableName, normalizedColumnName); if (existing == null) { - ColumnInfo columnInfo = new ColumnInfo(); - columnInfo.setDatasourceId(request.datasourceId()); - columnInfo.setTableName(normalizedTableName); - columnInfo.setColumnName(normalizedColumnName); - columnInfo.setColumnDescription( + ColumnSemantic columnSemantic = new ColumnSemantic(); + columnSemantic.setDatasourceId(request.datasourceId()); + columnSemantic.setTableName(normalizedTableName); + columnSemantic.setColumnName(normalizedColumnName); + columnSemantic.setColumnDescription( SemanticUtils.normalizeBlankToNull(request.columnDescription())); - columnInfo.setIsVisible(request.isVisible()); - columnInfo.setCreateTime(LocalDateTime.now()); - columnInfo.setUpdateTime(LocalDateTime.now()); - columnSemanticInfoMapper.insert(columnInfo); + columnSemantic.setIsVisible(request.isVisible()); + columnSemantic.setCreateTime(LocalDateTime.now()); + columnSemantic.setUpdateTime(LocalDateTime.now()); + columnSemanticMapper.insert(columnSemantic); return; } existing.setTableName(normalizedTableName); @@ -98,20 +101,19 @@ public void updateColumnSemantic(String tableName, ColumnSemanticUpdateRequest r SemanticUtils.normalizeBlankToNull(request.columnDescription())); existing.setIsVisible(request.isVisible()); existing.setUpdateTime(LocalDateTime.now()); - columnSemanticInfoMapper.update(existing); + columnSemanticMapper.update(existing); } @Override public void resetColumnSemantic(Integer datasourceId, String tableName, String columnName) { requireDatasource(datasourceId); - ColumnInfo existing = - columnSemanticInfoMapper.selectByDatasourceIdAndTableNameAndColumnName( + ColumnSemantic existing = + columnSemanticMapper.selectByDatasourceIdAndTableNameAndColumnName( datasourceId, tableName, columnName); if (existing == null) { throw new IllegalArgumentException("Column semantic metadata does not exist."); } - columnSemanticInfoMapper.deleteByDatasourceIdAndIds( - datasourceId, List.of(existing.getId())); + columnSemanticMapper.deleteByDatasourceIdAndIds(datasourceId, List.of(existing.getId())); } @Override @@ -131,14 +133,14 @@ public int resetColumnSemantics( columnName, "columnName"))) .collect(Collectors.toCollection(java.util.LinkedHashSet::new)); List matchedIds = - columnSemanticInfoMapper + columnSemanticMapper .selectByDatasourceIdAndTableName(datasourceId, normalizedTableName) .stream() .filter( column -> normalizedColumnNames.contains( normalizeKey(column.getColumnName()))) - .map(ColumnInfo::getId) + .map(ColumnSemantic::getId) .distinct() .toList(); if (matchedIds.isEmpty()) { @@ -150,7 +152,7 @@ public int resetColumnSemantics( + normalizedTableName + "."); } - return columnSemanticInfoMapper.deleteByDatasourceIdAndIds(datasourceId, matchedIds); + return columnSemanticMapper.deleteByDatasourceIdAndIds(datasourceId, matchedIds); } private void requireDatasource(Integer datasourceId) { @@ -160,21 +162,6 @@ private void requireDatasource(Integer datasourceId) { } } - private ColumnSemanticResponse mapResponse(ColumnInfo columnInfo) { - return new ColumnSemanticResponse( - columnInfo.getId(), - columnInfo.getColumnName(), - null, - columnInfo.getColumnDescription(), - null, - null, - columnInfo.getIsVisible(), - true, - Boolean.TRUE.equals(columnInfo.getIsVisible()), - null, - columnInfo.getUpdateTime()); - } - private String normalizeKey(String value) { return value.trim().toLowerCase(Locale.ROOT); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java index bafecff..96c3916 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticService.java @@ -17,9 +17,11 @@ */ package io.github.malonetalk.service.semantic.relation; -import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.PageResponse; import io.github.malonetalk.dto.semantic.BindLogicalTableRelationRequest; import io.github.malonetalk.dto.semantic.LogicalTableRelationResponse; +import io.github.malonetalk.dto.semantic.RelationCandidateColumnResponse; +import io.github.malonetalk.dto.semantic.RelationCandidateTableResponse; import io.github.malonetalk.dto.semantic.RelationSemanticPageQuery; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationEnabledRequest; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationRequest; @@ -41,4 +43,10 @@ boolean updateRelationSemanticEnabled( boolean deleteRelationSemantic(Integer datasourceId, String tableName, Integer relationId); int deleteRelationSemantics(Integer datasourceId, String tableName, List relationIds); + + PageResponse getCandidateTablePage( + RelationSemanticPageQuery query); + + PageResponse getCandidateColumnPage( + RelationSemanticPageQuery query); } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java index 4ebaf41..28f7273 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/relation/RelationSemanticServiceImpl.java @@ -20,22 +20,40 @@ import com.github.pagehelper.Page; import com.github.pagehelper.PageHelper; import io.github.malonetalk.common.SemanticConstants; -import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.convertor.LogicalTableRelationConverter; +import io.github.malonetalk.dto.PageResponse; import io.github.malonetalk.dto.semantic.BindLogicalTableRelationRequest; +import io.github.malonetalk.dto.semantic.ColumnSemanticPageQuery; import io.github.malonetalk.dto.semantic.LogicalTableRelationResponse; +import io.github.malonetalk.dto.semantic.RelationCandidateColumnResponse; +import io.github.malonetalk.dto.semantic.RelationCandidateTableResponse; import io.github.malonetalk.dto.semantic.RelationSemanticPageQuery; +import io.github.malonetalk.dto.semantic.TableSemanticPageQuery; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationEnabledRequest; import io.github.malonetalk.dto.semantic.UpdateLogicalTableRelationRequest; +import io.github.malonetalk.entity.Column; +import io.github.malonetalk.entity.ColumnSemantic; +import io.github.malonetalk.entity.Datasource; import io.github.malonetalk.entity.LogicalTableRelation; +import io.github.malonetalk.entity.TableSemantic; +import io.github.malonetalk.exception.SemanticSchemaException; +import io.github.malonetalk.infrastructure.SchemaReader; +import io.github.malonetalk.mapper.ColumnSemanticMapper; import io.github.malonetalk.mapper.LogicalTableRelationMapper; +import io.github.malonetalk.mapper.TableSemanticMapper; import io.github.malonetalk.service.DatasourceService; import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; +import java.util.HashMap; import java.util.List; +import java.util.Locale; +import java.util.Map; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +@Slf4j @Service @RequiredArgsConstructor public class RelationSemanticServiceImpl implements RelationSemanticService { @@ -43,6 +61,10 @@ public class RelationSemanticServiceImpl implements RelationSemanticService { private final DatasourceService datasourceService; private final LogicalTableRelationMapper logicalTableRelationMapper; private final LogicalTableRelationHelper logicalTableRelationHelper; + private final LogicalTableRelationConverter logicalTableRelationConverter; + private final TableSemanticMapper tableSemanticMapper; + private final ColumnSemanticMapper columnSemanticMapper; + private final SchemaReader schemaReader; @Override public PageResponse getRelationPage( @@ -71,7 +93,8 @@ public PageResponse getRelationPage( if (page.getTotal() == 0L) { return PageResponse.empty(pageNumber, pageSize); } - List items = page.stream().map(this::mapResponse).toList(); + List items = + page.stream().map(logicalTableRelationConverter::toResponse).toList(); return PageResponse.of(items, page.getTotal(), pageNumber, pageSize); } @@ -80,6 +103,12 @@ public PageResponse getRelationPage( public LogicalTableRelationResponse createRelationSemantic( String tableName, BindLogicalTableRelationRequest request) { requireDatasource(request.datasourceId()); + validateEndpoints( + request.datasourceId(), + tableName, + request.sourceColumnNames(), + request.targetTableName(), + request.targetColumnNames()); LogicalTableRelation relation = buildRelation(request.datasourceId(), tableName, request); ensureUniqueSourceKey( request.datasourceId(), @@ -87,7 +116,7 @@ public LogicalTableRelationResponse createRelationSemantic( relation.getSourceColumnSignature(), null); logicalTableRelationMapper.insert(relation); - return mapResponse(relation); + return logicalTableRelationConverter.toResponse(relation); } @Override @@ -95,6 +124,12 @@ public LogicalTableRelationResponse createRelationSemantic( public LogicalTableRelationResponse updateRelationSemantic( String tableName, UpdateLogicalTableRelationRequest request) { requireDatasource(request.datasourceId()); + validateEndpoints( + request.datasourceId(), + tableName, + request.sourceColumnNames(), + request.targetTableName(), + request.targetColumnNames()); LogicalTableRelation existing = requireRelation(request.datasourceId(), tableName, request.relationId()); applyRelationUpdate(existing, tableName, request); @@ -105,7 +140,7 @@ public LogicalTableRelationResponse updateRelationSemantic( existing.getId()); existing.setUpdateTime(LocalDateTime.now()); logicalTableRelationMapper.update(existing); - return mapResponse(existing); + return logicalTableRelationConverter.toResponse(existing); } @Override @@ -166,6 +201,108 @@ public int deleteRelationSemantics( datasourceId, normalizedTableName, relationIds); } + @Override + public PageResponse getCandidateTablePage( + RelationSemanticPageQuery query) { + requireDatasource(query.datasourceId()); + int pageNumber = PageResponse.resolvePage(query.page()); + int pageSize = PageResponse.resolvePageSize(query.pageSize()); + SemanticUtils.validateSortOrder(query.sortOrder()); + boolean sortDescending = + SemanticConstants.SORT_ORDER_DESC.equalsIgnoreCase(query.sortOrder()); + PageHelper.startPage(pageNumber, pageSize); + Page page = + (Page) + tableSemanticMapper.selectVisiblePageByDatasourceId( + new TableSemanticPageQuery( + query.datasourceId(), + pageNumber, + pageSize, + SemanticUtils.normalizeBlankToNull(query.keyword()), + query.sortOrder()), + sortDescending); + List items = + page.stream() + .map( + t -> + new RelationCandidateTableResponse( + t.getTableName(), + SemanticUtils.normalizeBlankToNull(t.getDomain()), + SemanticUtils.normalizeBlankToNull( + t.getTableDescription()))) + .toList(); + return PageResponse.of(items, page.getTotal(), pageNumber, pageSize); + } + + @Override + public PageResponse getCandidateColumnPage( + RelationSemanticPageQuery query) { + requireDatasource(query.datasourceId()); + String normalizedTableName = + logicalTableRelationHelper.normalizeTableName(query.tableName(), "tableName"); + TableSemantic table = + tableSemanticMapper.selectByDatasourceIdAndTableName( + query.datasourceId(), normalizedTableName); + if (table == null || !Boolean.TRUE.equals(table.getIsVisible())) { + throw new IllegalArgumentException("表 " + normalizedTableName + " 不存在或不可见。"); + } + int pageNumber = PageResponse.resolvePage(query.page()); + int pageSize = PageResponse.resolvePageSize(query.pageSize()); + SemanticUtils.validateSortOrder(query.sortOrder()); + boolean sortDescending = + SemanticConstants.SORT_ORDER_DESC.equalsIgnoreCase(query.sortOrder()); + PageHelper.startPage(pageNumber, pageSize); + Page page = + (Page) + columnSemanticMapper.selectVisiblePageByDatasourceIdAndTableName( + new ColumnSemanticPageQuery( + query.datasourceId(), + normalizedTableName, + pageNumber, + pageSize, + SemanticUtils.normalizeBlankToNull(query.keyword()), + query.sortOrder()), + sortDescending); + + // 物理列信息补充 + Map physicalByKey = new HashMap<>(); + try { + for (Column col : + schemaReader.getTableSchema( + datasourceService.findById(query.datasourceId()), + normalizedTableName)) { + physicalByKey.put(col.columnName().toLowerCase(Locale.ROOT), col); + } + } catch (SchemaReader.SchemaReadException e) { + log.warn("无法读取表 {} 的物理列信息: {}", normalizedTableName, e.getMessage()); + } + + List items = + page.stream() + .map( + c -> { + Column phys = + physicalByKey.get( + c.getColumnName().toLowerCase(Locale.ROOT)); + return new RelationCandidateColumnResponse( + c.getColumnName(), + SemanticUtils.normalizeBlankToNull( + c.getColumnDescription()), + phys != null ? buildTypeText(phys) : null, + phys != null ? phys.primaryKey() : null); + }) + .toList(); + return PageResponse.of(items, page.getTotal(), pageNumber, pageSize); + } + + private static String buildTypeText(Column physicalColumn) { + StringBuilder sb = new StringBuilder(physicalColumn.typeName()); + if (physicalColumn.columnSize() > 0) { + sb.append("(").append(physicalColumn.columnSize()).append(")"); + } + return sb.toString(); + } + private void requireDatasource(Integer datasourceId) { SemanticUtils.requireDatasourceId(datasourceId); if (datasourceService.findById(datasourceId) == null) { @@ -173,6 +310,114 @@ private void requireDatasource(Integer datasourceId) { } } + private void validateEndpoints( + Integer datasourceId, + String sourceTableName, + List sourceColumnNames, + String targetTableName, + List targetColumnNames) { + Datasource datasource = datasourceService.findById(datasourceId); + if (datasource == null) { + throw new IllegalArgumentException("Datasource does not exist: " + datasourceId); + } + String normalizedSourceTableName = + logicalTableRelationHelper.normalizeTableName(sourceTableName, "sourceTableName"); + String normalizedTargetTableName = + logicalTableRelationHelper.normalizeTableName(targetTableName, "targetTableName"); + List normalizedSourceColumnNames = + logicalTableRelationHelper.normalizeColumnNames( + sourceColumnNames, "sourceColumnNames"); + List normalizedTargetColumnNames = + logicalTableRelationHelper.normalizeColumnNames( + targetColumnNames, "targetColumnNames"); + TableSemantic sourceTable = + tableSemanticMapper.selectByDatasourceIdAndTableName( + datasourceId, normalizedSourceTableName); + if (sourceTable == null || !Boolean.TRUE.equals(sourceTable.getIsVisible())) { + throw new IllegalArgumentException("源表 " + normalizedSourceTableName + " 不存在或不可见。"); + } + TableSemantic targetTable = + tableSemanticMapper.selectByDatasourceIdAndTableName( + datasourceId, normalizedTargetTableName); + if (targetTable == null || !Boolean.TRUE.equals(targetTable.getIsVisible())) { + throw new IllegalArgumentException("目标表 " + normalizedTargetTableName + " 不存在或不可见。"); + } + for (String columnName : normalizedSourceColumnNames) { + ColumnSemantic col = + columnSemanticMapper.selectByDatasourceIdAndTableNameAndColumnName( + datasourceId, normalizedSourceTableName, columnName); + if (col == null || !Boolean.TRUE.equals(col.getIsVisible())) { + throw new IllegalArgumentException( + "源列 " + normalizedSourceTableName + "." + columnName + " 不存在或不可见。"); + } + } + for (String columnName : normalizedTargetColumnNames) { + ColumnSemantic col = + columnSemanticMapper.selectByDatasourceIdAndTableNameAndColumnName( + datasourceId, normalizedTargetTableName, columnName); + if (col == null || !Boolean.TRUE.equals(col.getIsVisible())) { + throw new IllegalArgumentException( + "目标列 " + normalizedTargetTableName + "." + columnName + " 不存在或不可见。"); + } + } + + validatePhysicalSchema( + datasource, + normalizedSourceTableName, + normalizedSourceColumnNames, + normalizedTargetTableName, + normalizedTargetColumnNames); + } + + private void validatePhysicalSchema( + Datasource datasource, + String sourceTableName, + List sourceColumnNames, + String targetTableName, + List targetColumnNames) { + Map sourceSchema = loadSchema(datasource, sourceTableName); + Map targetSchema = loadSchema(datasource, targetTableName); + if (sourceSchema.isEmpty()) { + throw new SemanticSchemaException( + "Physical source table does not exist: " + sourceTableName); + } + if (targetSchema.isEmpty()) { + throw new SemanticSchemaException( + "Physical target table does not exist: " + targetTableName); + } + for (String columnName : sourceColumnNames) { + if (!sourceSchema.containsKey(columnName.toLowerCase(Locale.ROOT))) { + throw new IllegalArgumentException( + "Physical source column does not exist: " + + sourceTableName + + "." + + columnName); + } + } + for (String columnName : targetColumnNames) { + if (!targetSchema.containsKey(columnName.toLowerCase(Locale.ROOT))) { + throw new IllegalArgumentException( + "Physical target column does not exist: " + + targetTableName + + "." + + columnName); + } + } + } + + private Map loadSchema(Datasource datasource, String tableName) { + Map physicalByKey = new HashMap<>(); + try { + for (Column column : schemaReader.getTableSchema(datasource, tableName)) { + physicalByKey.put(column.columnName().toLowerCase(Locale.ROOT), column); + } + } catch (SchemaReader.SchemaReadException e) { + throw new SemanticSchemaException( + "Failed to read physical schema for table " + tableName, e); + } + return physicalByKey; + } + private LogicalTableRelation buildRelation( Integer datasourceId, String tableName, BindLogicalTableRelationRequest request) { LogicalTableRelation relation = new LogicalTableRelation(); @@ -254,33 +499,4 @@ private LogicalTableRelation requireRelation( } return relation; } - - private LogicalTableRelationResponse mapResponse(LogicalTableRelation relation) { - List sourceColumns = - logicalTableRelationHelper.fromJson( - relation.getSourceColumnNamesJson(), "sourceColumnNames"); - List targetColumns = - logicalTableRelationHelper.fromJson( - relation.getTargetColumnNamesJson(), "targetColumnNames"); - return new LogicalTableRelationResponse( - relation.getId(), - logicalTableRelationHelper.buildRelationKey( - relation.getSourceTableName(), - sourceColumns, - relation.getTargetTableName(), - targetColumns), - relation.getDatasourceId(), - LogicalTableRelationHelper.RELATION_SOURCE_LOGICAL, - relation.getSourceTableName(), - sourceColumns, - relation.getTargetTableName(), - targetColumns, - relation.getRelationType(), - relation.getDescription(), - relation.getIsEnabled(), - relation.getIsEnabled(), - null, - relation.getCreateTime(), - relation.getUpdateTime()); - } } diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticService.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticService.java index 83e8717..997d7d6 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticService.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticService.java @@ -17,11 +17,10 @@ */ package io.github.malonetalk.service.semantic.table; -import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.dto.PageResponse; import io.github.malonetalk.dto.semantic.TableSemanticPageQuery; import io.github.malonetalk.dto.semantic.TableSemanticResponse; import io.github.malonetalk.dto.semantic.TableSemanticUpdateRequest; -import io.github.malonetalk.entity.TableInfo; import java.util.List; public interface TableSemanticService { @@ -30,8 +29,6 @@ public interface TableSemanticService { List listAvailableDomains(Integer datasourceId); - List listTableInfosByDatasourceId(Integer datasourceId); - void updateTableSemantic(TableSemanticUpdateRequest request); void resetTableSemantic(Integer datasourceId, String tableName); diff --git a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java index e1223e3..57dea23 100644 --- a/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java +++ b/data-agent-backend/src/main/java/io/github/malonetalk/service/semantic/table/TableSemanticServiceImpl.java @@ -20,12 +20,13 @@ import com.github.pagehelper.Page; import com.github.pagehelper.PageHelper; import io.github.malonetalk.common.SemanticConstants; -import io.github.malonetalk.dto.pagination.PageResponse; +import io.github.malonetalk.convertor.TableSemanticConverter; +import io.github.malonetalk.dto.PageResponse; import io.github.malonetalk.dto.semantic.TableSemanticPageQuery; import io.github.malonetalk.dto.semantic.TableSemanticResponse; import io.github.malonetalk.dto.semantic.TableSemanticUpdateRequest; -import io.github.malonetalk.entity.TableInfo; -import io.github.malonetalk.mapper.TableInfoMapper; +import io.github.malonetalk.entity.TableSemantic; +import io.github.malonetalk.mapper.TableSemanticMapper; import io.github.malonetalk.service.DatasourceService; import io.github.malonetalk.utils.SemanticUtils; import java.time.LocalDateTime; @@ -41,7 +42,8 @@ public class TableSemanticServiceImpl implements TableSemanticService { private final DatasourceService datasourceService; - private final TableInfoMapper tableInfoMapper; + private final TableSemanticMapper tableSemanticMapper; + private final TableSemanticConverter tableSemanticConverter; @Override public PageResponse getTablePage(TableSemanticPageQuery query) { @@ -55,9 +57,9 @@ public PageResponse getTablePage(TableSemanticPageQuery q boolean sortDescending = SemanticConstants.SORT_ORDER_DESC.equalsIgnoreCase(query.sortOrder()); PageHelper.startPage(pageNumber, pageSize); - Page page = - (Page) - tableInfoMapper.selectPageByDatasourceId( + Page page = + (Page) + tableSemanticMapper.selectPageByDatasourceId( new TableSemanticPageQuery( query.datasourceId(), pageNumber, @@ -65,7 +67,8 @@ public PageResponse getTablePage(TableSemanticPageQuery q SemanticUtils.normalizeBlankToNull(query.keyword()), query.sortOrder()), sortDescending); - List responses = page.stream().map(this::mapResponse).toList(); + List responses = + page.stream().map(tableSemanticConverter::toResponse).toList(); long total = page.getTotal(); return PageResponse.of(responses, total, pageNumber, pageSize); } @@ -76,8 +79,8 @@ public List listAvailableDomains(Integer datasourceId) { if (datasourceService.findById(datasourceId) == null) { return List.of(); } - return tableInfoMapper.selectByDatasourceId(datasourceId).stream() - .map(TableInfo::getDomain) + return tableSemanticMapper.selectByDatasourceId(datasourceId).stream() + .map(TableSemantic::getDomain) .filter(domain -> domain != null && !domain.isBlank()) .map(String::trim) .distinct() @@ -85,33 +88,24 @@ public List listAvailableDomains(Integer datasourceId) { .toList(); } - @Override - public List listTableInfosByDatasourceId(Integer datasourceId) { - SemanticUtils.requireDatasourceId(datasourceId); - if (datasourceService.findById(datasourceId) == null) { - return List.of(); - } - return tableInfoMapper.selectByDatasourceId(datasourceId); - } - @Override public void updateTableSemantic(TableSemanticUpdateRequest request) { requireDatasource(request.datasourceId()); String normalizedTableName = SemanticUtils.requireName(request.tableName(), "tableName"); - TableInfo existing = - tableInfoMapper.selectByDatasourceIdAndTableName( + TableSemantic existing = + tableSemanticMapper.selectByDatasourceIdAndTableName( request.datasourceId(), normalizedTableName); if (existing == null) { - TableInfo tableInfo = new TableInfo(); - tableInfo.setDatasourceId(request.datasourceId()); - tableInfo.setTableName(normalizedTableName); - tableInfo.setTableDescription( + TableSemantic tableSemantic = new TableSemantic(); + tableSemantic.setDatasourceId(request.datasourceId()); + tableSemantic.setTableName(normalizedTableName); + tableSemantic.setTableDescription( SemanticUtils.normalizeBlankToNull(request.tableDescription())); - tableInfo.setDomain(SemanticUtils.normalizeBlankToNull(request.domain())); - tableInfo.setIsVisible(request.isVisible()); - tableInfo.setCreateTime(LocalDateTime.now()); - tableInfo.setUpdateTime(LocalDateTime.now()); - tableInfoMapper.insert(tableInfo); + tableSemantic.setDomain(SemanticUtils.normalizeBlankToNull(request.domain())); + tableSemantic.setIsVisible(request.isVisible()); + tableSemantic.setCreateTime(LocalDateTime.now()); + tableSemantic.setUpdateTime(LocalDateTime.now()); + tableSemanticMapper.insert(tableSemantic); return; } existing.setTableName(normalizedTableName); @@ -120,19 +114,20 @@ public void updateTableSemantic(TableSemanticUpdateRequest request) { existing.setDomain(SemanticUtils.normalizeBlankToNull(request.domain())); existing.setIsVisible(request.isVisible()); existing.setUpdateTime(LocalDateTime.now()); - tableInfoMapper.update(existing); + tableSemanticMapper.update(existing); } @Override public void resetTableSemantic(Integer datasourceId, String tableName) { requireDatasource(datasourceId); String normalizedTableName = SemanticUtils.requireName(tableName, "tableName"); - TableInfo existing = - tableInfoMapper.selectByDatasourceIdAndTableName(datasourceId, normalizedTableName); + TableSemantic existing = + tableSemanticMapper.selectByDatasourceIdAndTableName( + datasourceId, normalizedTableName); if (existing == null) { throw new IllegalArgumentException("Table semantic metadata does not exist."); } - tableInfoMapper.deleteByDatasourceIdAndIds(datasourceId, List.of(existing.getId())); + tableSemanticMapper.deleteByDatasourceIdAndIds(datasourceId, List.of(existing.getId())); } @Override @@ -146,12 +141,12 @@ public int resetTableSemantics(Integer datasourceId, List tableNames) { .map(this::normalizeName) .collect(Collectors.toCollection(java.util.LinkedHashSet::new)); List matchedIds = - tableInfoMapper.selectByDatasourceId(datasourceId).stream() + tableSemanticMapper.selectByDatasourceId(datasourceId).stream() .filter( table -> normalizedNames.contains( normalizeName(table.getTableName()))) - .map(TableInfo::getId) + .map(TableSemantic::getId) .distinct() .toList(); if (matchedIds.isEmpty()) { @@ -163,7 +158,7 @@ public int resetTableSemantics(Integer datasourceId, List tableNames) { + datasourceId + "."); } - return tableInfoMapper.deleteByDatasourceIdAndIds(datasourceId, matchedIds); + return tableSemanticMapper.deleteByDatasourceIdAndIds(datasourceId, matchedIds); } private void requireDatasource(Integer datasourceId) { @@ -173,20 +168,6 @@ private void requireDatasource(Integer datasourceId) { } } - private TableSemanticResponse mapResponse(TableInfo tableInfo) { - return new TableSemanticResponse( - tableInfo.getId(), - tableInfo.getTableName(), - tableInfo.getDomain(), - null, - tableInfo.getTableDescription(), - tableInfo.getIsVisible(), - true, - Boolean.TRUE.equals(tableInfo.getIsVisible()), - null, - tableInfo.getUpdateTime()); - } - private String normalizeName(String value) { return SemanticUtils.requireName(value, "tableName").toLowerCase(Locale.ROOT); } diff --git a/data-agent-backend/src/main/resources/application.properties b/data-agent-backend/src/main/resources/application.properties index 50ae6d6..02d19fb 100644 --- a/data-agent-backend/src/main/resources/application.properties +++ b/data-agent-backend/src/main/resources/application.properties @@ -16,7 +16,7 @@ spring.datasource.hikari.connection-timeout=20000 # MyBatis Configuration mybatis.mapper-locations=classpath:mapper/*.xml -mybatis.type-aliases-package=com.malone.entity +mybatis.type-aliases-package=io.github.malonetalk.entity mybatis.configuration.map-underscore-to-camel-case=true mybatis.configuration.log-impl=org.apache.ibatis.logging.stdout.StdOutImpl diff --git a/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml b/data-agent-backend/src/main/resources/mapper/ColumnSemanticMapper.xml similarity index 80% rename from data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml rename to data-agent-backend/src/main/resources/mapper/ColumnSemanticMapper.xml index eb3cf09..e2337f3 100644 --- a/data-agent-backend/src/main/resources/mapper/ColumnSemanticInfoMapper.xml +++ b/data-agent-backend/src/main/resources/mapper/ColumnSemanticMapper.xml @@ -1,8 +1,8 @@ - + - + @@ -43,6 +43,24 @@ + + - + INSERT INTO column_info ( datasource_id, table_name, column_name, column_description, is_visible, create_time, update_time @@ -62,7 +80,7 @@ ) - + UPDATE column_info table_name = #{tableName}, diff --git a/data-agent-backend/src/main/resources/mapper/TableInfoMapper.xml b/data-agent-backend/src/main/resources/mapper/TableSemanticMapper.xml similarity index 79% rename from data-agent-backend/src/main/resources/mapper/TableInfoMapper.xml rename to data-agent-backend/src/main/resources/mapper/TableSemanticMapper.xml index c308e2f..cb376ef 100644 --- a/data-agent-backend/src/main/resources/mapper/TableInfoMapper.xml +++ b/data-agent-backend/src/main/resources/mapper/TableSemanticMapper.xml @@ -1,8 +1,8 @@ - + - + @@ -35,6 +35,23 @@ + + - + INSERT INTO table_info ( table_name, table_description, domain, datasource_id, is_visible, create_time, update_time @@ -53,7 +70,7 @@ ) - + UPDATE table_info table_name = #{tableName},