org.apache.maven.plugins
- maven-jar-plugin
- 3.3.0
-
-
-
- org.apache.ossie.converter.polaris.OsiPolarisConverter
-
-
-
+ maven-shade-plugin
+ ${maven-shade-plugin.version}
+
+
+ package
+
+ shade
+
+
+ false
+ true
+ all
+
+
+ org.apache.ossie.converter.polaris.OsiPolarisConverter
+
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+
+
+
+
+
+
diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/IcebergTypeMapper.java b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/IcebergTypeMapper.java
similarity index 98%
rename from converters/polaris/src/main/java/org/apache/ossie/converter/polaris/IcebergTypeMapper.java
rename to converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/IcebergTypeMapper.java
index 48f82ef2..e7ed0b7b 100644
--- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/IcebergTypeMapper.java
+++ b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/IcebergTypeMapper.java
@@ -24,8 +24,8 @@
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
-import org.apache.ossie.converter.polaris.model.OsiModel.CustomExtension;
-import org.apache.ossie.converter.polaris.model.OsiModel.Field;
+import org.apache.ossie.model.CustomExtension;
+import org.apache.ossie.model.Field;
import java.io.IOException;
import java.util.Locale;
diff --git a/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiModelParser.java b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiModelParser.java
new file mode 100644
index 00000000..fd64862d
--- /dev/null
+++ b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiModelParser.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ossie.converter.polaris;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.apache.ossie.model.OsiSchema;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Parses an Ossie YAML document into the schema-generated {@link OsiSchema} model.
+ */
+public class OsiModelParser {
+
+ private final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());
+
+ /**
+ * Parse an Ossie YAML file from the given path.
+ */
+ public OsiSchema parse(Path yamlPath) throws IOException {
+ try (InputStream inputStream = Files.newInputStream(yamlPath)) {
+ return yamlMapper.readValue(inputStream, OsiSchema.class);
+ }
+ }
+
+ /**
+ * Parse an Ossie YAML file from an input stream.
+ */
+ public OsiSchema parse(InputStream inputStream) {
+ try {
+ return yamlMapper.readValue(inputStream, OsiSchema.class);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Invalid Ossie YAML", e);
+ }
+ }
+}
diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiPolarisConverter.java b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiPolarisConverter.java
similarity index 94%
rename from converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiPolarisConverter.java
rename to converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiPolarisConverter.java
index c9eb50d7..2b7169a0 100644
--- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiPolarisConverter.java
+++ b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiPolarisConverter.java
@@ -19,7 +19,7 @@
package org.apache.ossie.converter.polaris;
-import org.apache.ossie.converter.polaris.model.OsiModel;
+import org.apache.ossie.model.OsiSchema;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -117,9 +117,9 @@ public static void main(String[] args) throws Exception {
private static void doImport(PolarisClient client, String outputFile) throws Exception {
PolarisImporter importer = new PolarisImporter(client);
- OsiModel model = importer.importCatalog();
+ OsiSchema model = importer.importCatalog();
- if (model.getSemanticModels().isEmpty()) {
+ if (model.getSemanticModel().isEmpty()) {
System.err.println("Warning: no tables found in catalog.");
}
@@ -141,9 +141,9 @@ private static void doExport(PolarisClient client, String inputFile) throws Exce
}
OsiModelParser parser = new OsiModelParser();
- OsiModel model = parser.parse(Paths.get(inputFile));
+ OsiSchema model = parser.parse(Paths.get(inputFile));
- if (model.getSemanticModels().isEmpty()) {
+ if (model.getSemanticModel().isEmpty()) {
System.err.println("Error: no semantic_model found in " + inputFile);
System.exit(1);
}
@@ -151,7 +151,7 @@ private static void doExport(PolarisClient client, String inputFile) throws Exce
PolarisExporter exporter = new PolarisExporter(client);
exporter.exportModel(model);
- System.out.println("Exported " + model.getSemanticModels().size()
+ System.out.println("Exported " + model.getSemanticModel().size()
+ " semantic model(s) to Polaris catalog.");
}
diff --git a/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiYamlGenerator.java b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiYamlGenerator.java
new file mode 100644
index 00000000..3aa93388
--- /dev/null
+++ b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiYamlGenerator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ossie.converter.polaris;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
+import org.apache.ossie.model.OsiSchema;
+
+/**
+ * Generates Ossie YAML from the schema-generated {@link OsiSchema} model.
+ */
+public class OsiYamlGenerator {
+
+ private final ObjectMapper yamlMapper;
+
+ public OsiYamlGenerator() {
+ YAMLFactory yamlFactory = new YAMLFactory()
+ .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER)
+ .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)
+ .enable(YAMLGenerator.Feature.LITERAL_BLOCK_STYLE);
+ this.yamlMapper = new ObjectMapper(yamlFactory);
+ }
+
+ /**
+ * Generate an Ossie YAML string from a model.
+ */
+ public String generate(OsiSchema model) {
+ try {
+ return yamlMapper.writeValueAsString(model);
+ } catch (JsonProcessingException e) {
+ throw new IllegalArgumentException("Unable to generate Ossie YAML", e);
+ }
+ }
+}
diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisClient.java b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisClient.java
similarity index 100%
rename from converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisClient.java
rename to converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisClient.java
diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java
similarity index 90%
rename from converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java
rename to converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java
index 49095637..816b243a 100644
--- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java
+++ b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java
@@ -24,8 +24,10 @@
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
-import org.apache.ossie.converter.polaris.model.OsiModel;
-import org.apache.ossie.converter.polaris.model.OsiModel.*;
+import org.apache.ossie.model.Dataset;
+import org.apache.ossie.model.Field;
+import org.apache.ossie.model.OsiSchema;
+import org.apache.ossie.model.SemanticModel;
import java.io.IOException;
import java.util.Collections;
@@ -56,8 +58,8 @@ public PolarisExporter(PolarisClient client) {
* Export the Ossie model to the Polaris catalog.
* Each semantic model becomes a namespace, and each dataset becomes a table.
*/
- public void exportModel(OsiModel model) throws IOException, InterruptedException {
- for (SemanticModel sm : model.getSemanticModels()) {
+ public void exportModel(OsiSchema model) throws IOException, InterruptedException {
+ for (SemanticModel sm : model.getSemanticModel()) {
exportSemanticModel(sm);
}
}
@@ -155,23 +157,24 @@ private ObjectNode buildSchema(Dataset dataset) {
* models authored before datatype support.
*/
private JsonNode inferIcebergType(Field field, AtomicInteger nextNestedId) {
+ String fieldDatatype = field.getDatatype() == null ? null : field.getDatatype().toString();
JsonNode exactType = IcebergTypeMapper.exactIcebergType(field, objectMapper);
if (exactType != null) {
String extensionDatatype = IcebergTypeMapper.toOssieDatatype(exactType);
- if (field.getDatatype() != null
- && !Objects.equals(field.getDatatype(), extensionDatatype)) {
+ if (fieldDatatype != null
+ && !Objects.equals(fieldDatatype, extensionDatatype)) {
IcebergTypeMapper.warn(
field.getName(),
- "datatype '" + field.getDatatype() + "' conflicts with exact Iceberg type '"
+ "datatype '" + fieldDatatype + "' conflicts with exact Iceberg type '"
+ IcebergTypeMapper.displayIcebergType(exactType)
+ "'; preserving the POLARIS extension value");
}
return IcebergTypeMapper.prepareExactTypeForExport(exactType, nextNestedId);
}
- JsonNode portableType = IcebergTypeMapper.toDefaultIcebergType(field.getDatatype());
+ JsonNode portableType = IcebergTypeMapper.toDefaultIcebergType(fieldDatatype);
if (portableType != null) {
- if ("Decimal".equals(field.getDatatype())) {
+ if ("Decimal".equals(fieldDatatype)) {
IcebergTypeMapper.warn(
field.getName(),
"Ossie datatype 'Decimal' has no precision or scale; using decimal(18, 2)");
@@ -179,8 +182,8 @@ private JsonNode inferIcebergType(Field field, AtomicInteger nextNestedId) {
return portableType;
}
- if (field.getDatatype() != null) {
- if ("Opaque".equals(field.getDatatype())) {
+ if (fieldDatatype != null) {
+ if ("Opaque".equals(fieldDatatype)) {
IcebergTypeMapper.warn(
field.getName(),
"Ossie datatype 'Opaque' has no exact Iceberg type in a POLARIS extension; "
@@ -188,7 +191,7 @@ private JsonNode inferIcebergType(Field field, AtomicInteger nextNestedId) {
} else {
IcebergTypeMapper.warn(
field.getName(),
- "unrecognized Ossie datatype '" + field.getDatatype()
+ "unrecognized Ossie datatype '" + fieldDatatype
+ "'; using legacy inference");
}
}
@@ -205,7 +208,8 @@ private JsonNode inferIcebergType(Field field, AtomicInteger nextNestedId) {
}
// A time role is only a fallback; it never overrides an explicit datatype.
- if (field.isTime()) {
+ if (field.getDimension() != null
+ && Boolean.TRUE.equals(field.getDimension().getIsTime())) {
return TextNode.valueOf("timestamptz");
}
diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java
similarity index 83%
rename from converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java
rename to converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java
index b00e1133..4750c1fa 100644
--- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java
+++ b/converters/java/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java
@@ -21,8 +21,14 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.ossie.converter.polaris.model.OsiModel;
-import org.apache.ossie.converter.polaris.model.OsiModel.*;
+import org.apache.ossie.model.CustomExtension;
+import org.apache.ossie.model.Dataset;
+import org.apache.ossie.model.DialectExpression;
+import org.apache.ossie.model.Dimension;
+import org.apache.ossie.model.Expression;
+import org.apache.ossie.model.Field;
+import org.apache.ossie.model.OsiSchema;
+import org.apache.ossie.model.SemanticModel;
import java.io.IOException;
import java.util.ArrayList;
@@ -34,7 +40,7 @@
*
* Reads namespaces and tables from a Polaris catalog via the Iceberg REST API,
* maps Iceberg table schemas to Ossie datasets and fields, and produces a complete
- * {@link OsiModel}.
+ * {@link OsiSchema}.
*/
public class PolarisImporter {
@@ -48,8 +54,8 @@ public PolarisImporter(PolarisClient client) {
* Import all tables from all namespaces in the catalog into an Ossie model.
* Each namespace becomes a separate semantic model.
*/
- public OsiModel importCatalog() throws IOException, InterruptedException {
- OsiModel model = new OsiModel();
+ public OsiSchema importCatalog() throws IOException, InterruptedException {
+ OsiSchema model = new OsiSchema();
model.setVersion("0.2.0.dev0");
List> namespaces = client.listNamespaces();
@@ -57,7 +63,7 @@ public OsiModel importCatalog() throws IOException, InterruptedException {
for (List namespace : namespaces) {
SemanticModel sm = importNamespace(namespace);
if (sm != null && !sm.getDatasets().isEmpty()) {
- model.getSemanticModels().add(sm);
+ model.getSemanticModel().add(sm);
}
}
@@ -91,7 +97,7 @@ public SemanticModel importNamespace(List namespace) throws IOException,
/**
* Map an Iceberg table's metadata to an Ossie dataset.
*/
- private Dataset mapTableToDataset(List namespace, String tableName, JsonNode tableMetadata) {
+ Dataset mapTableToDataset(List namespace, String tableName, JsonNode tableMetadata) {
Dataset dataset = new Dataset();
dataset.setName(tableName);
@@ -122,7 +128,9 @@ private Dataset mapTableToDataset(List namespace, String tableName, Json
// Store Polaris-specific table properties as custom extension
JsonNode properties = metadata.get("properties");
if (properties != null && properties.isObject() && properties.size() > 0) {
- CustomExtension ext = new CustomExtension("COMMON", properties.toString());
+ CustomExtension ext = new CustomExtension();
+ ext.setVendorName("COMMON");
+ ext.setData(properties.toString());
dataset.setCustomExtensions(Collections.singletonList(ext));
}
}
@@ -184,15 +192,23 @@ private Field mapColumnToField(JsonNode column) {
Field field = new Field();
field.setName(name);
- field.setDatatype(datatype);
+ if (datatype != null) {
+ field.setDatatype(Field.DataType.fromValue(datatype));
+ }
// The expression is just the column name (direct mapping)
- DialectExpression expr = new DialectExpression("ANSI_SQL", name);
- field.setExpressions(Collections.singletonList(expr));
+ DialectExpression expr = new DialectExpression();
+ expr.setDialect(DialectExpression.Dialect.ANSI_SQL);
+ expr.setExpression(name);
+ Expression expression = new Expression();
+ expression.setDialects(Collections.singletonList(expr));
+ field.setExpression(expression);
// Preserve the converter's existing temporal-role classification.
if (IcebergTypeMapper.isTemporalDatatype(datatype)) {
- field.setTime(true);
+ Dimension dimension = new Dimension();
+ dimension.setIsTime(true);
+ field.setDimension(dimension);
}
// DataType is intentionally logical and loses physical details such as
@@ -201,8 +217,10 @@ private Field mapColumnToField(JsonNode column) {
if (icebergTypeNode != null) {
ObjectNode extensionData = client.getObjectMapper().createObjectNode();
extensionData.set(IcebergTypeMapper.ICEBERG_TYPE_KEY, icebergTypeNode.deepCopy());
- field.setCustomExtensions(Collections.singletonList(
- new CustomExtension(IcebergTypeMapper.POLARIS_VENDOR, extensionData.toString())));
+ CustomExtension extension = new CustomExtension();
+ extension.setVendorName(IcebergTypeMapper.POLARIS_VENDOR);
+ extension.setData(extensionData.toString());
+ field.setCustomExtensions(Collections.singletonList(extension));
}
// Add type information as description
diff --git a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/IcebergTypeMapperTest.java b/converters/java/polaris/src/test/java/org/apache/ossie/converter/polaris/IcebergTypeMapperTest.java
similarity index 100%
rename from converters/polaris/src/test/java/org/apache/ossie/converter/polaris/IcebergTypeMapperTest.java
rename to converters/java/polaris/src/test/java/org/apache/ossie/converter/polaris/IcebergTypeMapperTest.java
diff --git a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java b/converters/java/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java
similarity index 80%
rename from converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java
rename to converters/java/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java
index 456bdb2c..af18d63e 100644
--- a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java
+++ b/converters/java/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java
@@ -22,8 +22,15 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
-import org.apache.ossie.converter.polaris.model.OsiModel;
-import org.apache.ossie.converter.polaris.model.OsiModel.*;
+import org.apache.ossie.model.CustomExtension;
+import org.apache.ossie.model.Dataset;
+import org.apache.ossie.model.DialectExpression;
+import org.apache.ossie.model.Dimension;
+import org.apache.ossie.model.Expression;
+import org.apache.ossie.model.Field;
+import org.apache.ossie.model.OsiSchema;
+import org.apache.ossie.model.Relationship;
+import org.apache.ossie.model.SemanticModel;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -42,6 +49,7 @@ class OsiPolarisConverterTest {
+ "semantic_model:\n"
+ " - name: test_model\n"
+ " description: A test model\n"
+ + " ai_context: sales analysis\n"
+ " datasets:\n"
+ " - name: orders\n"
+ " source: catalog.ns.orders\n"
@@ -107,14 +115,15 @@ class OsiPolarisConverterTest {
@Test
void testParseMinimalModel() {
OsiModelParser parser = new OsiModelParser();
- OsiModel model = parser.parse(
+ OsiSchema model = parser.parse(
new ByteArrayInputStream(MINIMAL_MODEL.getBytes(StandardCharsets.UTF_8)));
assertEquals("0.2.0.dev0", model.getVersion());
- assertEquals(1, model.getSemanticModels().size());
+ assertEquals(1, model.getSemanticModel().size());
- SemanticModel sm = model.getSemanticModels().get(0);
+ SemanticModel sm = model.getSemanticModel().get(0);
assertEquals("test_model", sm.getName());
+ assertEquals("sales analysis", sm.getAiContext());
assertEquals(2, sm.getDatasets().size());
assertEquals(1, sm.getRelationships().size());
assertEquals(1, sm.getMetrics().size());
@@ -123,10 +132,10 @@ void testParseMinimalModel() {
@Test
void testParseDatasetFields() {
OsiModelParser parser = new OsiModelParser();
- OsiModel model = parser.parse(
+ OsiSchema model = parser.parse(
new ByteArrayInputStream(MINIMAL_MODEL.getBytes(StandardCharsets.UTF_8)));
- Dataset orders = model.getSemanticModels().get(0).getDatasets().get(0);
+ Dataset orders = model.getSemanticModel().get(0).getDatasets().get(0);
assertEquals("orders", orders.getName());
assertEquals("catalog.ns.orders", orders.getSource());
assertEquals(3, orders.getFields().size());
@@ -134,30 +143,31 @@ void testParseDatasetFields() {
Field computed = orders.getFields().get(1);
assertEquals("total_amount", computed.getName());
- assertEquals("Decimal", computed.getDatatype());
- assertEquals("quantity * unit_price", computed.getExpressions().get(0).getExpression());
+ assertEquals(Field.DataType.DECIMAL, computed.getDatatype());
+ assertEquals("quantity * unit_price",
+ computed.getExpression().getDialects().get(0).getExpression());
}
@Test
void testParseTimeDimension() {
OsiModelParser parser = new OsiModelParser();
- OsiModel model = parser.parse(
+ OsiSchema model = parser.parse(
new ByteArrayInputStream(MINIMAL_MODEL.getBytes(StandardCharsets.UTF_8)));
- Dataset orders = model.getSemanticModels().get(0).getDatasets().get(0);
+ Dataset orders = model.getSemanticModel().get(0).getDatasets().get(0);
Field orderDate = orders.getFields().get(2);
assertEquals("order_date", orderDate.getName());
- assertEquals("Date", orderDate.getDatatype());
- assertTrue(orderDate.isTime());
+ assertEquals(Field.DataType.DATE, orderDate.getDatatype());
+ assertTrue(orderDate.getDimension().getIsTime());
}
@Test
void testParseRelationship() {
OsiModelParser parser = new OsiModelParser();
- OsiModel model = parser.parse(
+ OsiSchema model = parser.parse(
new ByteArrayInputStream(MINIMAL_MODEL.getBytes(StandardCharsets.UTF_8)));
- Relationship rel = model.getSemanticModels().get(0).getRelationships().get(0);
+ Relationship rel = model.getSemanticModel().get(0).getRelationships().get(0);
assertEquals("orders_to_customer", rel.getName());
assertEquals("orders", rel.getFrom());
assertEquals("customer", rel.getTo());
@@ -170,33 +180,34 @@ void testParseRelationship() {
@Test
void testYamlGenerationRoundTrip() {
OsiModelParser parser = new OsiModelParser();
- OsiModel model = parser.parse(
+ OsiSchema model = parser.parse(
new ByteArrayInputStream(MINIMAL_MODEL.getBytes(StandardCharsets.UTF_8)));
OsiYamlGenerator generator = new OsiYamlGenerator();
String yaml = generator.generate(model);
// Verify key elements are present in generated YAML
- assertTrue(yaml.contains("version: \"0.2.0.dev0\""));
+ assertTrue(yaml.contains("version:"));
assertTrue(yaml.contains("name: test_model"));
assertTrue(yaml.contains("name: orders"));
assertTrue(yaml.contains("source: catalog.ns.orders"));
- assertTrue(yaml.contains("primary_key: [order_id]"));
+ assertTrue(yaml.contains("primary_key:"));
assertTrue(yaml.contains("name: total_amount"));
assertTrue(yaml.contains("datatype: Decimal"));
assertTrue(yaml.contains("datatype: Date"));
assertTrue(yaml.contains("is_time: true"));
assertTrue(yaml.contains("name: orders_to_customer"));
- assertTrue(yaml.contains("from_columns: [customer_id]"));
+ assertTrue(yaml.contains("from_columns:"));
assertTrue(yaml.contains("name: total_revenue"));
assertTrue(yaml.contains("SUM(orders.total_amount)"));
// Re-parse the generated YAML to verify it's valid
- OsiModel reparsed = parser.parse(
+ OsiSchema reparsed = parser.parse(
new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)));
- assertEquals(1, reparsed.getSemanticModels().size());
- assertEquals("test_model", reparsed.getSemanticModels().get(0).getName());
- assertEquals(2, reparsed.getSemanticModels().get(0).getDatasets().size());
+ assertEquals(1, reparsed.getSemanticModel().size());
+ assertEquals("test_model", reparsed.getSemanticModel().get(0).getName());
+ assertEquals("sales analysis", reparsed.getSemanticModel().get(0).getAiContext());
+ assertEquals(2, reparsed.getSemanticModel().get(0).getDatasets().size());
}
// -- Exporter tests (Iceberg schema generation) -------------------------
@@ -204,13 +215,13 @@ void testYamlGenerationRoundTrip() {
@Test
void testExporterBuildCreateTableRequest() throws Exception {
OsiModelParser parser = new OsiModelParser();
- OsiModel model = parser.parse(
+ OsiSchema model = parser.parse(
new ByteArrayInputStream(MINIMAL_MODEL.getBytes(StandardCharsets.UTF_8)));
PolarisClient client = new PolarisClient("http://localhost:8181", "test_catalog");
PolarisExporter exporter = new PolarisExporter(client);
- Dataset orders = model.getSemanticModels().get(0).getDatasets().get(0);
+ Dataset orders = model.getSemanticModel().get(0).getDatasets().get(0);
String json = exporter.buildCreateTableRequest(orders);
ObjectMapper mapper = new ObjectMapper();
@@ -290,27 +301,31 @@ void testImporterMapTableToDataset() throws Exception {
PolarisClient client = new FakePolarisClient(tableMetadata);
PolarisImporter importer = new PolarisImporter(client);
- OsiModel model = importer.importCatalog();
+ OsiSchema model = importer.importCatalog();
- Dataset ds = model.getSemanticModels().get(0).getDatasets().get(0);
+ Dataset ds = model.getSemanticModel().get(0).getDatasets().get(0);
assertEquals("test_table", ds.getName());
assertEquals("test_catalog.test_ns.test_table", ds.getSource());
assertEquals(Collections.singletonList("id"), ds.getPrimaryKey());
assertEquals(6, ds.getFields().size());
- assertEquals("Integer", ds.getFields().get(0).getDatatype());
- assertEquals("String", ds.getFields().get(1).getDatatype());
- assertEquals("DateTimeTz", ds.getFields().get(2).getDatatype());
- assertTrue(ds.getFields().get(2).isTime());
- assertEquals("Decimal", ds.getFields().get(3).getDatatype());
- assertEquals("DateTime", ds.getFields().get(4).getDatatype());
- assertTrue(ds.getFields().get(4).isTime());
- assertEquals("Opaque", ds.getFields().get(5).getDatatype());
+ assertEquals(Field.DataType.INTEGER, ds.getFields().get(0).getDatatype());
+ assertEquals(Field.DataType.STRING, ds.getFields().get(1).getDatatype());
+ assertEquals(Field.DataType.DATE_TIME_TZ, ds.getFields().get(2).getDatatype());
+ assertTrue(ds.getFields().get(2).getDimension().getIsTime());
+ assertEquals(Field.DataType.DECIMAL, ds.getFields().get(3).getDatatype());
+ assertEquals(Field.DataType.DATE_TIME, ds.getFields().get(4).getDatatype());
+ assertTrue(ds.getFields().get(4).getDimension().getIsTime());
+ assertEquals(Field.DataType.OPAQUE, ds.getFields().get(5).getDatatype());
JsonNode tagsExtension = mapper.readTree(
ds.getFields().get(5).getCustomExtensions().get(0).getData());
assertEquals("list", tagsExtension.path("iceberg_type").path("type").asText());
assertEquals(99, tagsExtension.path("iceberg_type").path("element-id").asInt());
+ assertEquals(DialectExpression.Dialect.ANSI_SQL,
+ ds.getFields().get(0).getExpression().getDialects().get(0).getDialect());
+ assertEquals("test_user", new ObjectMapper().readTree(
+ ds.getCustomExtensions().get(0).getData()).get("owner").asText());
// Generate YAML and verify
OsiYamlGenerator generator = new OsiYamlGenerator();
@@ -318,7 +333,7 @@ void testImporterMapTableToDataset() throws Exception {
assertTrue(yaml.contains("name: test_table"));
assertTrue(yaml.contains("source: test_catalog.test_ns.test_table"));
- assertTrue(yaml.contains("primary_key: [id]"));
+ assertTrue(yaml.contains("primary_key:"));
assertTrue(yaml.contains("name: id"));
assertTrue(yaml.contains("name: name"));
assertTrue(yaml.contains("name: created_at"));
@@ -334,9 +349,9 @@ void testImporterMapTableToDataset() throws Exception {
// Reparse the generated Ossie YAML and export it again. Exact physical
// distinctions survive, while nested IDs are regenerated for the new schema.
- OsiModel reparsed = new OsiModelParser().parse(
+ OsiSchema reparsed = new OsiModelParser().parse(
new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)));
- Dataset reparsedDataset = reparsed.getSemanticModels().get(0).getDatasets().get(0);
+ Dataset reparsedDataset = reparsed.getSemanticModel().get(0).getDatasets().get(0);
JsonNode exportedSchema = mapper.readTree(
new PolarisExporter(client).buildCreateTableRequest(reparsedDataset)).path("schema");
JsonNode exportedFields = exportedSchema.path("fields");
@@ -355,24 +370,24 @@ void testDatatypePrecedenceAndLegacyFallbacks() throws Exception {
ds.setSource("cat.ns.precedence_test");
Field typedStringWithTimeRole = makeField("typed_string", true);
- typedStringWithTimeRole.setDatatype("String");
+ typedStringWithTimeRole.setDatatype(Field.DataType.STRING);
Field typedLocalTimestamp = makeField("local_timestamp", false);
- typedLocalTimestamp.setDatatype("DateTime");
+ typedLocalTimestamp.setDatatype(Field.DataType.DATE_TIME);
Field legacyUuid = makeField("legacy_value", false);
legacyUuid.setDescription("Iceberg type: uuid (optional)");
Field exactConflict = makeField("exact_conflict", false);
- exactConflict.setDatatype("String");
+ exactConflict.setDatatype(Field.DataType.STRING);
exactConflict.setCustomExtensions(Collections.singletonList(
- new CustomExtension("POLARIS", "{\"iceberg_type\":\"int\"}")));
+ makeExtension("POLARIS", "{\"iceberg_type\":\"int\"}")));
Field opaqueWithoutExtension = makeField("opaque_id", false);
- opaqueWithoutExtension.setDatatype("Opaque");
+ opaqueWithoutExtension.setDatatype(Field.DataType.OPAQUE);
Field decimalWithoutExtension = makeField("amount", false);
- decimalWithoutExtension.setDatatype("Decimal");
+ decimalWithoutExtension.setDatatype(Field.DataType.DECIMAL);
ds.setFields(java.util.Arrays.asList(
typedStringWithTimeRole,
@@ -410,7 +425,7 @@ void testDatatypePrecedenceAndLegacyFallbacks() throws Exception {
@Test
void testTypeInference() throws Exception {
// Test that the exporter correctly infers Iceberg types from field names
- OsiModel model = new OsiModel();
+ OsiSchema model = new OsiSchema();
model.setVersion("0.2.0.dev0");
SemanticModel sm = new SemanticModel();
@@ -433,7 +448,7 @@ void testTypeInference() throws Exception {
ds.setFields(fields);
sm.setDatasets(Collections.singletonList(ds));
- model.setSemanticModels(Collections.singletonList(sm));
+ model.setSemanticModel(Collections.singletonList(sm));
PolarisClient client = new PolarisClient("http://localhost:8181", "cat");
PolarisExporter exporter = new PolarisExporter(client);
@@ -457,11 +472,11 @@ void testTypeInference() throws Exception {
void testEmptyModel() {
String emptyYaml = "version: \"0.2.0.dev0\"\n";
OsiModelParser parser = new OsiModelParser();
- OsiModel model = parser.parse(
+ OsiSchema model = parser.parse(
new ByteArrayInputStream(emptyYaml.getBytes(StandardCharsets.UTF_8)));
assertEquals("0.2.0.dev0", model.getVersion());
- assertTrue(model.getSemanticModels().isEmpty());
+ assertTrue(model.getSemanticModel().isEmpty());
}
// -- Helpers ------------------------------------------------------------
@@ -469,8 +484,12 @@ void testEmptyModel() {
private Field makeField(String name, boolean isTime) {
Field f = new Field();
f.setName(name);
- f.setExpressions(Collections.singletonList(new DialectExpression("ANSI_SQL", name)));
- f.setTime(isTime);
+ f.setExpression(makeExpression(name));
+ if (isTime) {
+ Dimension dimension = new Dimension();
+ dimension.setIsTime(true);
+ f.setDimension(dimension);
+ }
return f;
}
@@ -497,4 +516,21 @@ public JsonNode loadTable(List namespace, String tableName) {
return tableMetadata;
}
}
+
+ private Expression makeExpression(String expressionValue) {
+ DialectExpression dialectExpression = new DialectExpression();
+ dialectExpression.setDialect(DialectExpression.Dialect.ANSI_SQL);
+ dialectExpression.setExpression(expressionValue);
+
+ Expression expression = new Expression();
+ expression.setDialects(Collections.singletonList(dialectExpression));
+ return expression;
+ }
+
+ private CustomExtension makeExtension(String vendorName, String data) {
+ CustomExtension extension = new CustomExtension();
+ extension.setVendorName(vendorName);
+ extension.setData(data);
+ return extension;
+ }
}
diff --git a/converters/java/pom.xml b/converters/java/pom.xml
new file mode 100644
index 00000000..7e0f6edc
--- /dev/null
+++ b/converters/java/pom.xml
@@ -0,0 +1,56 @@
+
+
+
+
+ 4.0.0
+
+
+ org.apache
+ apache
+ 39
+
+
+
+ org.apache.ossie
+ ossie-java-converters
+ 0.1.0-SNAPSHOT
+ pom
+
+ Apache Ossie Java Converters
+ Reactor for the shared Ossie Java model and Java converters
+
+
+ ossie-common
+ polaris
+ salesforce
+
+
+
+
+
+ org.apache.ossie
+ ossie-common
+ ${project.version}
+
+
+
+
diff --git a/converters/salesforce/README.md b/converters/java/salesforce/README.md
similarity index 83%
rename from converters/salesforce/README.md
rename to converters/java/salesforce/README.md
index d49ea2f0..e1b2de07 100644
--- a/converters/salesforce/README.md
+++ b/converters/java/salesforce/README.md
@@ -19,7 +19,7 @@
# Apache Ossie Salesforce Converter
-A two-way converter between [Ossie semantic models](../../core-spec/spec.md) and [Salesforce Semantic Model](https://developer.salesforce.com/docs/data/semantic-layer/guide/salesforce-semantic-model-schema.html).
+A two-way converter between [Ossie semantic models](../../../core-spec/spec.md) and [Salesforce Semantic Model](https://developer.salesforce.com/docs/data/semantic-layer/guide/salesforce-semantic-model-schema.html).
This converter supports conversion in both directions between Ossie YAML and
Salesforce Semantic Model JSON. Unmapped Salesforce properties are preserved in
@@ -28,33 +28,32 @@ Salesforce Semantic Model JSON. Unmapped Salesforce properties are preserved in
## Requirements
- **Java 17+**
-- **Maven 3.6+** — required to build the jar
+- **Maven 3.9.12+** — required to build the Java reactor
## Building
Build the executable jar from source:
```bash
-mvn clean package
+cd converters/java
+mvn -pl salesforce -am clean package
```
-This produces a self-contained executable jar at `target/ossie-salesforce-converter-0.1.0-SNAPSHOT.jar` with all dependencies bundled.
+This produces a self-contained executable jar at
+`salesforce/target/ossie-salesforce-converter-0.1.0-SNAPSHOT-all.jar` with all
+dependencies bundled.
## Setup
-Both schemas must be obtained and placed under `src/main/resources/schemas/` before building, so they get bundled into the jar.
+The canonical Apache Ossie schema is copied automatically from `core-spec` at
+build time. The Salesforce schema must be obtained separately and placed under
+`salesforce/src/main/resources/schemas/` so it can be bundled into the jar.
### Salesforce Semantic Model Schema
1. Visit the [Salesforce Semantic Model Schema documentation](https://developer.salesforce.com/docs/data/semantic-layer/guide/salesforce-semantic-model-schema.html)
2. Copy the JSON schema content from the page
-3. Save it to `src/main/resources/schemas/salesforce-semantic-model-schema.json`
-
-### Apache Ossie Schema
-
-1. Visit the [Ossie schema on GitHub](https://github.com/apache/ossie/blob/main/core-spec/osi-schema.json)
-2. Copy the raw JSON contents
-3. Save it to `src/main/resources/schemas/osi-schema.json`
+3. Save it to `salesforce/src/main/resources/schemas/salesforce-semantic-model-schema.json`
## Usage
@@ -65,16 +64,16 @@ Both schemas must be obtained and placed under `src/main/resources/schemas/` bef
Convert a Salesforce Semantic Model JSON file to Ossie YAML format:
```bash
-java -jar target/ossie-salesforce-converter-0.1.0-SNAPSHOT.jar toOSI input.json
+java -jar salesforce/target/ossie-salesforce-converter-0.1.0-SNAPSHOT-all.jar toOSI input.json
# Output: Customer_Orders_Model.yaml (named after model's 'name' field)
# Created in the same directory as the input file
```
Example:
```bash
-java -jar target/ossie-salesforce-converter-0.1.0-SNAPSHOT.jar toOSI \
- src/test/resources/examples/salesforceToOsi.json
-# Output: src/test/resources/examples/Customer_Orders_Model.yaml
+java -jar salesforce/target/ossie-salesforce-converter-0.1.0-SNAPSHOT-all.jar toOSI \
+ salesforce/src/test/resources/examples/salesforceToOsi.json
+# Output: salesforce/src/test/resources/examples/Customer_Orders_Model.yaml
```
#### Export (Apache Ossie → Salesforce)
@@ -82,16 +81,16 @@ java -jar target/ossie-salesforce-converter-0.1.0-SNAPSHOT.jar toOSI \
Convert an Ossie YAML file to Salesforce Semantic Model JSON format:
```bash
-java -jar target/ossie-salesforce-converter-0.1.0-SNAPSHOT.jar toSF input.yaml
+java -jar salesforce/target/ossie-salesforce-converter-0.1.0-SNAPSHOT-all.jar toSF input.yaml
# Output: Customer_Orders_Model.json (named after model's 'apiName' field)
# Created in the same directory as the input file
```
Example:
```bash
-java -jar target/ossie-salesforce-converter-0.1.0-SNAPSHOT.jar toSF \
- src/test/resources/examples/osiToSalesforce.yaml
-# Output: src/test/resources/examples/Customer_Orders_Model.json
+java -jar salesforce/target/ossie-salesforce-converter-0.1.0-SNAPSHOT-all.jar toSF \
+ salesforce/src/test/resources/examples/osiToSalesforce.yaml
+# Output: salesforce/src/test/resources/examples/Customer_Orders_Model.json
```
### Programmatic API
@@ -134,6 +133,9 @@ osiToSf.convert(Paths.get("input/model.yaml"), Paths.get("output/"));
- **Bidirectional** - Supports both directions, with direction-specific limits documented below
- **Supports Ossie Specification v0.2.0.dev0**
+Ossie metrics currently have no Salesforce export mapping and are omitted in
+the Ossie-to-Salesforce direction.
+
## Mapping Reference
### Import (Salesforce → Apache Ossie)
@@ -274,17 +276,17 @@ dimensions.
**GenericMappingEngine** — Path-based property mapping using `mappings.yaml` configuration
-**CustomExtensionHandler** — Preserves unmapped Salesforce properties in Ossie's `custom_extensions` for lossless bi-directional conversion
+**CustomExtensionHandler** — Preserves supported unmapped Salesforce properties in Ossie's `custom_extensions` for round-trip conversion
**SchemaValidator** — Validates input against JSON schemas before conversion
## Examples
See the test suite for sample models demonstrating various features:
-- `src/test/resources/examples/osiToSalesforce.yaml` - Ossie model example
-- `src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java` - Ossie to Salesforce conversion tests
-- `src/test/java/org/apache/ossie/SalesforceToOsiConverterTest.java` - Salesforce to Ossie conversion tests
+- `salesforce/src/test/resources/examples/osiToSalesforce.yaml` - Ossie model example
+- `salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java` - Ossie to Salesforce conversion tests
+- `salesforce/src/test/java/org/apache/ossie/SalesforceToOsiConverterTest.java` - Salesforce to Ossie conversion tests
## License
-Apache License 2.0 — see [LICENSE](../../LICENSE).
+Apache License 2.0 — see [LICENSE](../../../LICENSE).
diff --git a/converters/salesforce/pom.xml b/converters/java/salesforce/pom.xml
similarity index 92%
rename from converters/salesforce/pom.xml
rename to converters/java/salesforce/pom.xml
index 89c5b03f..e0c8fdca 100644
--- a/converters/salesforce/pom.xml
+++ b/converters/java/salesforce/pom.xml
@@ -24,15 +24,13 @@
4.0.0
- org.apache
- apache
- 39
-
+ org.apache.ossie
+ ossie-java-converters
+ 0.1.0-SNAPSHOT
+ ../pom.xml
- org.apache.ossie
ossie-salesforce-converter
- 0.1.0-SNAPSHOT
jar
Apache Ossie Salesforce Converter
@@ -55,6 +53,11 @@
+
+ org.apache.ossie
+ ossie-common
+
+
com.fasterxml.jackson.core
jackson-databind
@@ -100,7 +103,7 @@
Copy the canonical Ossie core-spec JSON schema into this module's
classpath under /schemas/osi-schema.json so SchemaValidator can
load it at runtime. Single source of truth lives in
- ../../core-spec/osi-schema.json; do not maintain a duplicate
+ ../../../core-spec/osi-schema.json; do not maintain a duplicate
copy in this module's src/main/resources.
-->
@@ -118,7 +121,7 @@
${project.build.outputDirectory}/schemas
- ${project.basedir}/../../core-spec
+ ${project.basedir}/../../../core-spec
osi-schema.json
@@ -141,6 +144,8 @@
false
+ true
+ all
org.apache.ossie.app.OsiSalesforceConverter
diff --git a/converters/salesforce/src/main/java/org/apache/ossie/app/OsiSalesforceConverter.java b/converters/java/salesforce/src/main/java/org/apache/ossie/app/OsiSalesforceConverter.java
similarity index 100%
rename from converters/salesforce/src/main/java/org/apache/ossie/app/OsiSalesforceConverter.java
rename to converters/java/salesforce/src/main/java/org/apache/ossie/app/OsiSalesforceConverter.java
diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java b/converters/java/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java
similarity index 100%
rename from converters/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java
rename to converters/java/salesforce/src/main/java/org/apache/ossie/converter/AbstractConverter.java
diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConversionDirection.java b/converters/java/salesforce/src/main/java/org/apache/ossie/converter/ConversionDirection.java
similarity index 100%
rename from converters/salesforce/src/main/java/org/apache/ossie/converter/ConversionDirection.java
rename to converters/java/salesforce/src/main/java/org/apache/ossie/converter/ConversionDirection.java
diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/Converter.java b/converters/java/salesforce/src/main/java/org/apache/ossie/converter/Converter.java
similarity index 100%
rename from converters/salesforce/src/main/java/org/apache/ossie/converter/Converter.java
rename to converters/java/salesforce/src/main/java/org/apache/ossie/converter/Converter.java
diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java b/converters/java/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java
similarity index 100%
rename from converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java
rename to converters/java/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java
diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java b/converters/java/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java
similarity index 100%
rename from converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java
rename to converters/java/salesforce/src/main/java/org/apache/ossie/converter/ConverterFactory.java
diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java b/converters/java/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java
similarity index 80%
rename from converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java
rename to converters/java/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java
index a6b5cff6..1f6cc37b 100644
--- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java
+++ b/converters/java/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java
@@ -26,8 +26,9 @@
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.ossie.converter.pipeline.*;
-import org.apache.ossie.converter.pipeline.*;
import org.apache.ossie.exception.ConversionException;
+import org.apache.ossie.model.OsiSchema;
+import org.apache.ossie.model.SemanticModel;
import org.apache.ossie.validator.SchemaValidator;
import java.util.ArrayList;
@@ -90,18 +91,19 @@ public List convert(String content) {
schemaValidator.validate(sourceData);
if (direction == ConversionDirection.OSI_TO_SALESFORCE) {
- return convertOsiToSalesforce(sourceData);
+ OsiSchema osiSchema = OsiModelBinding.fromRootMap(jsonMapper, sourceData);
+ return convertOsiToSalesforce(osiSchema);
} else {
return convertSalesforceToOsi(sourceData);
}
}
- private List convertOsiToSalesforce(Map osiRoot) {
- List