diff --git a/.gitignore b/.gitignore index 7fee9646f..abe2140ec 100644 --- a/.gitignore +++ b/.gitignore @@ -34,4 +34,5 @@ t.* Chart.lock yarn.lock package-lock.json -datavines-server/src/main/resources/static \ No newline at end of file +datavines-server/src/main/resources/static +design/ \ No newline at end of file diff --git a/datavines-dist/src/main/assembly/datavines-bin.xml b/datavines-dist/src/main/assembly/datavines-bin.xml index aaa30eabb..0f9e8e6f6 100644 --- a/datavines-dist/src/main/assembly/datavines-bin.xml +++ b/datavines-dist/src/main/assembly/datavines-bin.xml @@ -100,14 +100,7 @@ true io.datavines:datavines-common - io.datavines:datavines-spi - io.datavines:datavines-engine-spark-api - io.datavines:datavines-engine-spark-connector-jdbc - io.datavines:datavines-engine-spark-connector-mongodb io.datavines:datavines-engine-core - io.datavines:datavines-engine-common - io.datavines:datavines-engine-spark-transform-sql - io.datavines:datavines-engine-api com.mysql:mysql-connector-j org.apache.httpcomponents:httpclient org.apache.httpcomponents:httpcore @@ -127,14 +120,7 @@ true io.datavines:datavines-common - io.datavines:datavines-spi - io.datavines:datavines-engine-flink-api - io.datavines:datavines-engine-flink-connector-jdbc - io.datavines:datavines-engine-core - io.datavines:datavines-engine-common - io.datavines:datavines-engine-flink-transform-sql - io.datavines:datavines-engine-api com.mysql:mysql-connector-j org.apache.httpcomponents:httpclient org.apache.httpcomponents:httpcore diff --git a/datavines-engine/datavines-engine-api/pom.xml b/datavines-engine/datavines-engine-api/pom.xml deleted file mode 100644 index 2a7d06c7f..000000000 --- a/datavines-engine/datavines-engine-api/pom.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - datavines-engine - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - datavines-engine-api - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-common/.gitignore b/datavines-engine/datavines-engine-common/.gitignore deleted file mode 100644 index 5ff6309b7..000000000 --- a/datavines-engine/datavines-engine-common/.gitignore +++ /dev/null @@ -1,38 +0,0 @@ -target/ -!.mvn/wrapper/maven-wrapper.jar -!**/src/main/**/target/ -!**/src/test/**/target/ - -### IntelliJ IDEA ### -.idea/modules.xml -.idea/jarRepositories.xml -.idea/compiler.xml -.idea/libraries/ -*.iws -*.iml -*.ipr - -### Eclipse ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache - -### NetBeans ### -/nbproject/private/ -/nbbuild/ -/dist/ -/nbdist/ -/.nb-gradle/ -build/ -!**/src/main/**/build/ -!**/src/test/**/build/ - -### VS Code ### -.vscode/ - -### Mac OS ### -.DS_Store \ No newline at end of file diff --git a/datavines-engine/datavines-engine-common/pom.xml b/datavines-engine/datavines-engine-common/pom.xml deleted file mode 100644 index dd0f0521c..000000000 --- a/datavines-engine/datavines-engine-common/pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - 4.0.0 - - io.datavines - datavines-engine - 1.0.0-SNAPSHOT - - - datavines-engine-common - - - 8 - 8 - UTF-8 - - - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-config/pom.xml b/datavines-engine/datavines-engine-config/pom.xml index 8ad75a80c..be0bb0b2f 100644 --- a/datavines-engine/datavines-engine-config/pom.xml +++ b/datavines-engine/datavines-engine-config/pom.xml @@ -45,14 +45,8 @@ io.datavines - datavines-engine-api - ${project.version} - - - - io.datavines - datavines-engine-common + datavines-engine-core ${project.version} - \ No newline at end of file + diff --git a/datavines-engine/datavines-engine-core/pom.xml b/datavines-engine/datavines-engine-core/pom.xml index ba8cc3e09..02559bb7e 100644 --- a/datavines-engine/datavines-engine-core/pom.xml +++ b/datavines-engine/datavines-engine-core/pom.xml @@ -29,13 +29,4 @@ 4.0.0 datavines-engine-core - - - - io.datavines - datavines-engine-api - - - - - \ No newline at end of file + diff --git a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/EngineConstants.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/EngineConstants.java similarity index 100% rename from datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/EngineConstants.java rename to datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/EngineConstants.java diff --git a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/component/Component.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/component/Component.java similarity index 100% rename from datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/component/Component.java rename to datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/component/Component.java diff --git a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/engine/EngineExecutor.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/engine/EngineExecutor.java similarity index 100% rename from datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/engine/EngineExecutor.java rename to datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/engine/EngineExecutor.java diff --git a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/env/Execution.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/env/Execution.java similarity index 100% rename from datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/env/Execution.java rename to datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/env/Execution.java diff --git a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/env/RuntimeEnvironment.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/env/RuntimeEnvironment.java similarity index 100% rename from datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/env/RuntimeEnvironment.java rename to datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/env/RuntimeEnvironment.java diff --git a/datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/plugin/Plugin.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/plugin/Plugin.java similarity index 100% rename from datavines-engine/datavines-engine-api/src/main/java/io/datavines/engine/api/plugin/Plugin.java rename to datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/api/plugin/Plugin.java diff --git a/datavines-engine/datavines-engine-common/src/main/java/io/datavines/engine/common/utils/ParserUtils.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/common/utils/ParserUtils.java similarity index 100% rename from datavines-engine/datavines-engine-common/src/main/java/io/datavines/engine/common/utils/ParserUtils.java rename to datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/common/utils/ParserUtils.java diff --git a/datavines-engine/datavines-engine-common/src/main/java/io/datavines/engine/common/utils/QuoteIdentifier.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/common/utils/QuoteIdentifier.java similarity index 100% rename from datavines-engine/datavines-engine-common/src/main/java/io/datavines/engine/common/utils/QuoteIdentifier.java rename to datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/common/utils/QuoteIdentifier.java diff --git a/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/BaseDataVinesBootstrap.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/BaseDataVinesBootstrap.java index e1ebdfc79..f1d3c2e49 100644 --- a/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/BaseDataVinesBootstrap.java +++ b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/BaseDataVinesBootstrap.java @@ -16,27 +16,35 @@ */ package io.datavines.engine.core; +import io.datavines.common.config.DataVinesJobConfig; import io.datavines.common.entity.ProcessResult; import io.datavines.common.enums.ExecutionStatus; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -import java.util.List; import io.datavines.common.config.CheckResult; import io.datavines.common.config.ConfigRuntimeException; import io.datavines.engine.api.component.Component; import io.datavines.engine.api.env.Execution; import io.datavines.engine.api.env.RuntimeEnvironment; import io.datavines.engine.core.config.ConfigParser; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + public abstract class BaseDataVinesBootstrap { private static final Logger logger = LoggerFactory.getLogger(BaseDataVinesBootstrap.class); private Execution execution; + protected abstract RuntimeEnvironment createRuntimeEnvironment(DataVinesJobConfig config) throws Exception; + + protected abstract List createSources(DataVinesJobConfig config) throws Exception; + + protected abstract List createTransforms(DataVinesJobConfig config) throws Exception; + + protected abstract List createSinks(DataVinesJobConfig config) throws Exception; + public ProcessResult execute(String[] args) { if (args.length == 1) { try { @@ -65,12 +73,14 @@ public void stop() { private void parseConfigAndExecute(String configFile) throws Exception { ConfigParser configParser = new ConfigParser(configFile); - List sources = configParser.getSourcePlugins(); - List transforms = configParser.getTransformPlugins(); - List sinks = configParser.getSinkPlugins(); - execution = configParser.getRuntimeEnvironment().getExecution(); + DataVinesJobConfig jobConfig = configParser.getConfig(); + RuntimeEnvironment runtimeEnvironment = createRuntimeEnvironment(jobConfig); + List sources = createSources(jobConfig); + List transforms = createTransforms(jobConfig); + List sinks = createSinks(jobConfig); + execution = runtimeEnvironment.getExecution(); checkConfig(sources, transforms, sinks); - prepare(configParser.getRuntimeEnvironment(), sources, transforms, sinks); + prepare(runtimeEnvironment, sources, transforms, sinks); if (execution == null) { throw new Exception("can not create execution , please check the config"); } diff --git a/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/config/ConfigParser.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/config/ConfigParser.java index 5ec6dc7b4..ee2424a70 100644 --- a/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/config/ConfigParser.java +++ b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/config/ConfigParser.java @@ -16,23 +16,13 @@ */ package io.datavines.engine.core.config; -import io.datavines.common.config.Config; import io.datavines.common.config.ConfigRuntimeException; import io.datavines.common.config.DataVinesJobConfig; import io.datavines.common.config.EnvConfig; -import io.datavines.engine.api.component.Component; -import io.datavines.engine.api.env.RuntimeEnvironment; import io.datavines.engine.core.utils.JsonUtils; -import io.datavines.spi.PluginDiscovery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; - -import static io.datavines.engine.api.EngineConstants.PLUGIN_TYPE; -import static io.datavines.engine.api.EngineConstants.TYPE; - public class ConfigParser { private static final Logger logger = LoggerFactory.getLogger(ConfigParser.class); @@ -43,13 +33,10 @@ public class ConfigParser { private final EnvConfig envConfig; - private final RuntimeEnvironment env; - public ConfigParser(String configFile){ this.configFile = configFile; this.config = load(); this.envConfig = config.getEnvConfig(); - this.env = createRuntimeEnvironment(); } private DataVinesJobConfig load() { @@ -67,60 +54,11 @@ private DataVinesJobConfig load() { return config; } - private RuntimeEnvironment createRuntimeEnvironment() { - RuntimeEnvironment env = PluginDiscovery.getMultiKeyPluginDiscovery(RuntimeEnvironment.class, RuntimeEnvironment::getPluginNames) - - .getNewPlugin(envConfig.getEngine()); - Config config = new Config(envConfig.getConfig()); - config.put(TYPE, envConfig.getType()); - env.setConfig(config); - env.prepare(); - return env; - } - - public RuntimeEnvironment getRuntimeEnvironment() { - return env; - } - - public List getSourcePlugins() { - List sourcePluginList = new ArrayList<>(); - config.getSourceParameters().forEach(sourceConfig -> { - String pluginName = String.format("%s-%s-%s-source", envConfig.getEngine(), envConfig.getType(), sourceConfig.getPlugin()); - Component component = PluginDiscovery.getMultiKeyPluginDiscovery(Component.class, Component::getPluginNames) - - .getNewPlugin(pluginName); - sourceConfig.getConfig().put(PLUGIN_TYPE, sourceConfig.getType()); - component.setConfig(new Config(sourceConfig.getConfig())); - sourcePluginList.add(component); - }); - return sourcePluginList; - } - - public List getSinkPlugins() { - List sinkPluginList = new ArrayList<>(); - config.getSinkParameters().forEach(sinkConfig -> { - String pluginName = String.format("%s-%s-%s-sink", envConfig.getEngine(), envConfig.getType(), sinkConfig.getPlugin()); - Component component = PluginDiscovery.getMultiKeyPluginDiscovery(Component.class, Component::getPluginNames) - - .getNewPlugin(pluginName); - sinkConfig.getConfig().put(PLUGIN_TYPE, sinkConfig.getType()); - component.setConfig(new Config(sinkConfig.getConfig())); - sinkPluginList.add(component); - }); - return sinkPluginList; + public DataVinesJobConfig getConfig() { + return config; } - public List getTransformPlugins() { - List transformPluginList = new ArrayList<>(); - config.getTransformParameters().forEach(transformConfig -> { - String pluginName = String.format("%s-%s-%s-transform", envConfig.getEngine(), envConfig.getType(), transformConfig.getPlugin()); - Component component = PluginDiscovery.getMultiKeyPluginDiscovery(Component.class, Component::getPluginNames) - - .getNewPlugin(pluginName); - transformConfig.getConfig().put(PLUGIN_TYPE, transformConfig.getType()); - component.setConfig(new Config(transformConfig.getConfig())); - transformPluginList.add(component); - }); - return transformPluginList; + public EnvConfig getEnvConfig() { + return envConfig; } } diff --git a/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/enums/ConnectorType.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/enums/ConnectorType.java new file mode 100644 index 000000000..304b2f14b --- /dev/null +++ b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/enums/ConnectorType.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.engine.core.enums; + +public enum ConnectorType { + + JDBC("jdbc"), + MYSQL("mysql"), + FILE("file"), + MONGODB("mongodb"); + + private final String code; + + ConnectorType(String code) { + this.code = code; + } + + public static ConnectorType of(String code) { + for (ConnectorType connectorType : values()) { + if (connectorType.code.equalsIgnoreCase(code)) { + return connectorType; + } + } + throw new IllegalArgumentException("Unknown connector type: " + code); + } +} diff --git a/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/enums/EngineType.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/enums/EngineType.java new file mode 100644 index 000000000..965a4756f --- /dev/null +++ b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/enums/EngineType.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.engine.core.enums; + +public enum EngineType { + + LOCAL("local"), + SPARK("spark"), + FLINK("flink"), + LIVY("livy"); + + private final String code; + + EngineType(String code) { + this.code = code; + } + + public static EngineType of(String code) { + for (EngineType engineType : values()) { + if (engineType.code.equalsIgnoreCase(code)) { + return engineType; + } + } + throw new IllegalArgumentException("Unknown engine type: " + code); + } +} diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-core/src/main/java/io/datavines/engine/local/core/LocalDataVinesBootstrap.java b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/enums/ExecutionMode.java similarity index 63% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-core/src/main/java/io/datavines/engine/local/core/LocalDataVinesBootstrap.java rename to datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/enums/ExecutionMode.java index 639ded2a9..0303762ee 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-core/src/main/java/io/datavines/engine/local/core/LocalDataVinesBootstrap.java +++ b/datavines-engine/datavines-engine-core/src/main/java/io/datavines/engine/core/enums/ExecutionMode.java @@ -14,15 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.datavines.engine.local.core; +package io.datavines.engine.core.enums; -import io.datavines.engine.core.BaseDataVinesBootstrap; -import io.datavines.engine.local.api.utils.LoggerFactory; -import org.slf4j.Logger; +public enum ExecutionMode { -public class LocalDataVinesBootstrap extends BaseDataVinesBootstrap { + BATCH("batch"), + STREAM("stream"); - public LocalDataVinesBootstrap(Logger logger) { - LoggerFactory.setLogger(logger); + private final String code; + + ExecutionMode(String code) { + this.code = code; + } + + public static ExecutionMode of(String code) { + for (ExecutionMode executionMode : values()) { + if (executionMode.code.equalsIgnoreCase(code)) { + return executionMode; + } + } + return BATCH; } } diff --git a/datavines-engine/datavines-engine-executor/pom.xml b/datavines-engine/datavines-engine-executor/pom.xml index 89cb6ee7b..511570eac 100644 --- a/datavines-engine/datavines-engine-executor/pom.xml +++ b/datavines-engine/datavines-engine-executor/pom.xml @@ -33,7 +33,8 @@ io.datavines - datavines-engine-api + datavines-engine-core + ${project.version} @@ -52,4 +53,4 @@ - \ No newline at end of file + diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/pom.xml deleted file mode 100644 index 75e16ce58..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - datavines-engine-flink - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - - datavines-engine-flink-api - - - - io.datavines - datavines-engine-api - - - - io.datavines - datavines-common - - - - org.apache.flink - flink-streaming-java - - - - org.apache.flink - flink-table-api-java - - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - - - - diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment deleted file mode 100644 index 3ae30b0ad..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment +++ /dev/null @@ -1 +0,0 @@ -io.datavines.engine.flink.api.FlinkRuntimeEnvironment diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/pom.xml index 3394ce323..5bafb85ab 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-config/pom.xml @@ -32,17 +32,7 @@ io.datavines - datavines-engine-core - - - - io.datavines - datavines-engine-api - - - - io.datavines - datavines-engine-flink-api + datavines-engine-flink-core diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/pom.xml deleted file mode 100644 index e24409506..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/pom.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - - datavines-engine-flink - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - jar - - datavines-engine-flink-connector-jdbc - - - - io.datavines - datavines-engine-flink-api - - - - - org.apache.flink - flink-connector-jdbc - - - - org.apache.flink - flink-streaming-java - - - - org.apache.flink - flink-table-api-java - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - - - - io.datavines - datavines-engine-common - ${project.version} - - - - diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component deleted file mode 100644 index 05b69bc51..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component +++ /dev/null @@ -1,3 +0,0 @@ -io.datavines.engine.flink.jdbc.sink.JdbcSink -io.datavines.engine.flink.jdbc.sink.MySQLSink -io.datavines.engine.flink.jdbc.source.JdbcSource diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/pom.xml index bfe7f3ad8..152ff17cc 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/pom.xml @@ -35,84 +35,24 @@ datavines-engine-core - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + org.apache.flink + flink-connector-jdbc + - - - - + + org.apache.flink + flink-streaming-java + - - - - + + org.apache.flink + flink-table-api-java + - - - - + + org.apache.flink + flink-table-planner_${scala.binary.version} + - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/FlinkRuntimeEnvironment.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/FlinkRuntimeEnvironment.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/FlinkRuntimeEnvironment.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/FlinkRuntimeEnvironment.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/entity/FLinkColumnInfo.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/entity/FLinkColumnInfo.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/entity/FLinkColumnInfo.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/entity/FLinkColumnInfo.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamExecution.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamExecution.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamExecution.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamExecution.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamSource.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamSource.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamSource.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamTransform.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamTransform.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-api/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamTransform.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/api/stream/FlinkStreamTransform.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/core/FlinkDataVinesBootstrap.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/core/FlinkDataVinesBootstrap.java index 715716707..a227ecd6d 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/core/FlinkDataVinesBootstrap.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/core/FlinkDataVinesBootstrap.java @@ -16,11 +16,28 @@ */ package io.datavines.engine.flink.core; -import java.util.Base64; - +import io.datavines.common.config.Config; +import io.datavines.common.config.DataVinesJobConfig; +import io.datavines.engine.api.component.Component; +import io.datavines.engine.api.env.RuntimeEnvironment; import io.datavines.engine.core.BaseDataVinesBootstrap; +import io.datavines.engine.core.enums.ConnectorType; +import io.datavines.engine.flink.api.FlinkRuntimeEnvironment; +import io.datavines.engine.flink.api.stream.FlinkStreamSink; +import io.datavines.engine.flink.api.stream.FlinkStreamSource; +import io.datavines.engine.flink.jdbc.sink.JdbcSink; +import io.datavines.engine.flink.jdbc.sink.MySQLSink; +import io.datavines.engine.flink.jdbc.source.JdbcSource; +import io.datavines.engine.flink.transform.sql.SqlTransform; import lombok.extern.slf4j.Slf4j; +import java.util.Base64; +import java.util.List; +import java.util.stream.Collectors; + +import static io.datavines.engine.api.EngineConstants.PLUGIN_TYPE; +import static io.datavines.engine.api.EngineConstants.TYPE; + @Slf4j public class FlinkDataVinesBootstrap extends BaseDataVinesBootstrap { @@ -33,4 +50,61 @@ public static void main(String[] args) { bootstrap.execute(args); } } + + @Override + protected RuntimeEnvironment createRuntimeEnvironment(DataVinesJobConfig config) { + FlinkRuntimeEnvironment runtimeEnvironment = new FlinkRuntimeEnvironment(); + Config runtimeConfig = new Config(config.getEnvConfig().getConfig()); + runtimeConfig.put(TYPE, config.getEnvConfig().getType()); + runtimeEnvironment.setConfig(runtimeConfig); + runtimeEnvironment.prepare(); + return runtimeEnvironment; + } + + @Override + protected List createSources(DataVinesJobConfig config) { + return config.getSourceParameters().stream().map(sourceConfig -> { + FlinkStreamSource source; + switch (ConnectorType.of(sourceConfig.getPlugin())) { + case JDBC: + source = new JdbcSource(); + break; + default: + throw new IllegalArgumentException("Unsupported flink source plugin: " + sourceConfig.getPlugin()); + } + sourceConfig.getConfig().put(PLUGIN_TYPE, sourceConfig.getType()); + source.setConfig(new Config(sourceConfig.getConfig())); + return (Component) source; + }).collect(Collectors.toList()); + } + + @Override + protected List createTransforms(DataVinesJobConfig config) { + return config.getTransformParameters().stream().map(transformConfig -> { + SqlTransform transform = new SqlTransform(); + transformConfig.getConfig().put(PLUGIN_TYPE, transformConfig.getType()); + transform.setConfig(new Config(transformConfig.getConfig())); + return (Component) transform; + }).collect(Collectors.toList()); + } + + @Override + protected List createSinks(DataVinesJobConfig config) { + return config.getSinkParameters().stream().map(sinkConfig -> { + FlinkStreamSink sink; + switch (ConnectorType.of(sinkConfig.getPlugin())) { + case JDBC: + sink = new JdbcSink(); + break; + case MYSQL: + sink = new MySQLSink(); + break; + default: + throw new IllegalArgumentException("Unsupported flink sink plugin: " + sinkConfig.getPlugin()); + } + sinkConfig.getConfig().put(PLUGIN_TYPE, sinkConfig.getType()); + sink.setConfig(new Config(sinkConfig.getConfig())); + return (Component) sink; + }).collect(Collectors.toList()); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/JdbcSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/jdbc/sink/JdbcSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/JdbcSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/jdbc/sink/JdbcSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/MySQLSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/jdbc/sink/MySQLSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/sink/MySQLSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/jdbc/sink/MySQLSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/source/JdbcSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/jdbc/source/JdbcSource.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/source/JdbcSource.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/jdbc/source/JdbcSource.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/utils/DatabaseUrlReplacer.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/jdbc/utils/DatabaseUrlReplacer.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/utils/DatabaseUrlReplacer.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/jdbc/utils/DatabaseUrlReplacer.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/utils/FlinkTableUtils.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/jdbc/utils/FlinkTableUtils.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-connector-jdbc/src/main/java/io/datavines/engine/flink/jdbc/utils/FlinkTableUtils.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/jdbc/utils/FlinkTableUtils.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/java/io/datavines/engine/flink/transform/sql/SqlTransform.java b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/transform/sql/SqlTransform.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/java/io/datavines/engine/flink/transform/sql/SqlTransform.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-core/src/main/java/io/datavines/engine/flink/transform/sql/SqlTransform.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/pom.xml index 607c4e483..ed3f66e87 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-executor/pom.xml @@ -17,6 +17,7 @@ limitations under the License. --> + @@ -30,53 +31,17 @@ datavines-engine-flink-executor - - io.datavines - datavines-engine-api - - io.datavines datavines-engine-executor - - - - io.datavines - datavines-engine-core - - - - io.datavines - datavines-engine-flink-api - - - - io.datavines - datavines-common - - - - io.datavines - datavines-engine-flink-connector-jdbc ${project.version} - - - - - - - - - - - - io.datavines - datavines-engine-flink-transform-sql + datavines-engine-flink-core ${project.version} + diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/pom.xml deleted file mode 100644 index e64af8d44..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/pom.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - - datavines-engine-flink - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - jar - - datavines-engine-flink-transform-sql - - - - io.datavines - datavines-engine-flink-api - ${project.version} - - - - org.apache.flink - flink-streaming-java - - - - org.apache.flink - flink-table-api-java - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - - - diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component deleted file mode 100644 index ad7df8f28..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/datavines-engine-flink-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component +++ /dev/null @@ -1 +0,0 @@ -io.datavines.engine.flink.transform.sql.SqlTransform diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/pom.xml index 9087e533e..8a617680a 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-flink/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-flink/pom.xml @@ -31,12 +31,9 @@ pom - datavines-engine-flink-api datavines-engine-flink-core - datavines-engine-flink-config - datavines-engine-flink-transform-sql - datavines-engine-flink-connector-jdbc datavines-engine-flink-executor + datavines-engine-flink-config @@ -73,12 +70,6 @@ - - io.datavines - datavines-engine-api - ${project.version} - - org.apache.flink @@ -115,12 +106,6 @@ ${scope} - - io.datavines - datavines-engine-common - ${project.version} - - diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/pom.xml index e1eacdb0d..024a53413 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-livy/datavines-engine-livy-executor/pom.xml @@ -32,11 +32,6 @@ - - io.datavines - datavines-engine-api - - io.datavines datavines-engine-executor @@ -48,31 +43,6 @@ ${project.version} - - io.datavines - datavines-engine-spark-connector-jdbc - ${project.version} - - - - io.datavines - datavines-engine-spark-connector-mongodb - ${project.version} - - - - io.datavines - datavines-engine-spark-connector-file - ${project.version} - - - - io.datavines - datavines-engine-spark-transform-sql - ${project.version} - - - - \ No newline at end of file + diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-livy/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-livy/pom.xml index 7552052d1..35f4d62ef 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-livy/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-livy/pom.xml @@ -101,4 +101,4 @@ - \ No newline at end of file + diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/pom.xml deleted file mode 100644 index 2ef37d6f4..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/pom.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - datavines-engine-local - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - - datavines-engine-local-api - - - - io.datavines - datavines-engine-api - - - - io.datavines - datavines-connector-all - - - commons-io - commons-io - - - - - - - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment deleted file mode 100644 index 187696ce3..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment +++ /dev/null @@ -1 +0,0 @@ -io.datavines.engine.local.api.LocalRuntimeEnvironment diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/pom.xml deleted file mode 100644 index 7dbb3ff48..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/pom.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - datavines-engine-local - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - - datavines-engine-local-connector-file - - - - io.datavines - datavines-engine-local-api - ${project.version} - - - - com.h2database - h2 - 2.1.214 - - - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component deleted file mode 100644 index 57cfb3d7a..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component +++ /dev/null @@ -1,2 +0,0 @@ -io.datavines.engine.local.connector.LocalFileSink -io.datavines.engine.local.connector.LocalFileSource diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/pom.xml deleted file mode 100644 index 7346aa745..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - datavines-engine-local - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - - datavines-engine-local-connector-jdbc - - - - io.datavines - datavines-engine-local-api - ${project.version} - - - - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component deleted file mode 100644 index 86a5e2a07..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component +++ /dev/null @@ -1,3 +0,0 @@ -io.datavines.engine.local.connector.BaseJdbcSink -io.datavines.engine.local.connector.BaseJdbcSource -io.datavines.engine.local.connector.MySQLSink diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-core/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-core/pom.xml deleted file mode 100644 index 796a22e77..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-core/pom.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - datavines-engine-local - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - - datavines-engine-local-core - - - - io.datavines - datavines-engine-local-api - ${project.version} - - - - io.datavines - datavines-engine-core - ${project.version} - - - - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/pom.xml index de4e4e5f6..38b56db1c 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/pom.xml @@ -31,41 +31,40 @@ datavines-engine-local-executor - io.datavines - datavines-engine-api + datavines-engine-core + ${project.version} io.datavines datavines-engine-executor - - - - io.datavines - datavines-engine-local-core ${project.version} io.datavines - datavines-engine-local-connector-jdbc + datavines-connector-all ${project.version} + + + commons-io + commons-io + + - io.datavines - datavines-engine-local-connector-file - ${project.version} + com.h2database + h2 + 2.1.214 - io.datavines - datavines-engine-local-transform - ${project.version} + com.github.jsqlparser + jsqlparser - - \ No newline at end of file + diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalExecution.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/LocalExecution.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalExecution.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/LocalExecution.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalRuntimeEnvironment.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/LocalRuntimeEnvironment.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalRuntimeEnvironment.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/LocalRuntimeEnvironment.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/LocalSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/LocalSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/LocalSource.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalSource.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/LocalSource.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalTransform.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/LocalTransform.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/LocalTransform.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/LocalTransform.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/entity/ConnectionHolder.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/entity/ConnectionHolder.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/entity/ConnectionHolder.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/entity/ConnectionHolder.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/utils/FileUtils.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/utils/FileUtils.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/utils/FileUtils.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/utils/FileUtils.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/utils/LoggerFactory.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/utils/LoggerFactory.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-api/src/main/java/io/datavines/engine/local/api/utils/LoggerFactory.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/api/utils/LoggerFactory.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/BaseJdbcSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/BaseJdbcSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/BaseJdbcSource.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/LocalFileSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/LocalFileSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/LocalFileSource.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-file/src/main/java/io/datavines/engine/local/connector/LocalFileSource.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/LocalFileSource.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/MySQLSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/MySQLSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/MySQLSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/MySQLSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ActualValueDataSinkExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/ActualValueDataSinkExecutor.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ActualValueDataSinkExecutor.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/ActualValueDataSinkExecutor.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/BaseDataSinkExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/BaseDataSinkExecutor.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/BaseDataSinkExecutor.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/BaseDataSinkExecutor.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ErrorDataSinkExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/ErrorDataSinkExecutor.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ErrorDataSinkExecutor.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/ErrorDataSinkExecutor.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ISinkExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/ISinkExecutor.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ISinkExecutor.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/ISinkExecutor.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ProfileDataSinkExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/ProfileDataSinkExecutor.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ProfileDataSinkExecutor.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/ProfileDataSinkExecutor.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ValidateResultDataSinkExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/ValidateResultDataSinkExecutor.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-connector-jdbc/src/main/java/io/datavines/engine/local/connector/executor/ValidateResultDataSinkExecutor.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/connector/executor/ValidateResultDataSinkExecutor.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/core/LocalDataVinesBootstrap.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/core/LocalDataVinesBootstrap.java new file mode 100644 index 000000000..bec1bbd01 --- /dev/null +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/core/LocalDataVinesBootstrap.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.datavines.engine.local.core; + +import io.datavines.common.config.Config; +import io.datavines.common.config.DataVinesJobConfig; +import io.datavines.engine.api.component.Component; +import io.datavines.engine.api.env.RuntimeEnvironment; +import io.datavines.engine.core.BaseDataVinesBootstrap; +import io.datavines.engine.core.enums.ConnectorType; +import io.datavines.engine.local.api.LocalSink; +import io.datavines.engine.local.api.LocalSource; +import io.datavines.engine.local.api.LocalRuntimeEnvironment; +import io.datavines.engine.local.api.utils.LoggerFactory; +import io.datavines.engine.local.connector.*; +import io.datavines.engine.local.transform.sql.SqlTransform; +import org.slf4j.Logger; + +import java.util.List; +import java.util.stream.Collectors; + +import static io.datavines.engine.api.EngineConstants.PLUGIN_TYPE; +import static io.datavines.engine.api.EngineConstants.TYPE; + + +public class LocalDataVinesBootstrap extends BaseDataVinesBootstrap { + + public LocalDataVinesBootstrap(Logger logger) { + LoggerFactory.setLogger(logger); + } + + @Override + protected RuntimeEnvironment createRuntimeEnvironment(DataVinesJobConfig config) { + LocalRuntimeEnvironment runtimeEnvironment = new LocalRuntimeEnvironment(); + Config runtimeConfig = new Config(config.getEnvConfig().getConfig()); + runtimeConfig.put(TYPE, config.getEnvConfig().getType()); + runtimeEnvironment.setConfig(runtimeConfig); + runtimeEnvironment.prepare(); + return runtimeEnvironment; + } + + @Override + protected List createSources(DataVinesJobConfig config) { + return config.getSourceParameters().stream().map(sourceConfig -> { + LocalSource source; + switch (ConnectorType.of(sourceConfig.getPlugin())) { + case JDBC: + source = new BaseJdbcSource(); + break; + case FILE: + source = new LocalFileSource(); + break; + default: + throw new IllegalArgumentException("Unsupported local source plugin: " + sourceConfig.getPlugin()); + } + sourceConfig.getConfig().put(PLUGIN_TYPE, sourceConfig.getType()); + source.setConfig(new Config(sourceConfig.getConfig())); + return (Component) source; + }).collect(Collectors.toList()); + } + + @Override + protected List createTransforms(DataVinesJobConfig config) { + return config.getTransformParameters().stream().map(transformConfig -> { + SqlTransform transform = new SqlTransform(); + transformConfig.getConfig().put(PLUGIN_TYPE, transformConfig.getType()); + transform.setConfig(new Config(transformConfig.getConfig())); + return (Component) transform; + }).collect(Collectors.toList()); + } + + @Override + protected List createSinks(DataVinesJobConfig config) { + return config.getSinkParameters().stream().map(sinkConfig -> { + LocalSink sink; + switch (ConnectorType.of(sinkConfig.getPlugin())) { + case JDBC: + sink = new BaseJdbcSink(); + break; + case MYSQL: + sink = new MySQLSink(); + break; + case FILE: + sink = new LocalFileSink(); + break; + default: + throw new IllegalArgumentException("Unsupported local sink plugin: " + sinkConfig.getPlugin()); + } + sinkConfig.getConfig().put(PLUGIN_TYPE, sinkConfig.getType()); + sink.setConfig(new Config(sinkConfig.getConfig())); + return (Component) sink; + }).collect(Collectors.toList()); + } +} diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/ActualValueExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/transform/sql/ActualValueExecutor.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/ActualValueExecutor.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/transform/sql/ActualValueExecutor.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/ExpectedValueExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/transform/sql/ExpectedValueExecutor.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/ExpectedValueExecutor.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/transform/sql/ExpectedValueExecutor.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/ITransformExecutor.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/transform/sql/ITransformExecutor.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/ITransformExecutor.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/transform/sql/ITransformExecutor.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/SqlTransform.java b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/transform/sql/SqlTransform.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/java/io/datavines/engine/local/transform/sql/SqlTransform.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-executor/src/main/java/io/datavines/engine/local/transform/sql/SqlTransform.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/pom.xml deleted file mode 100644 index 2126a13e2..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/pom.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - datavines-engine-local - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - - datavines-engine-local-transform - - - - io.datavines - datavines-engine-local-api - ${project.version} - - - - com.github.jsqlparser - jsqlparser - - - - - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component deleted file mode 100644 index 70dc5c0a3..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/datavines-engine-local-transform/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component +++ /dev/null @@ -1 +0,0 @@ -io.datavines.engine.local.transform.sql.SqlTransform diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-local/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-local/pom.xml index 33fa93ae3..83d6e9931 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-local/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-local/pom.xml @@ -31,13 +31,8 @@ pom - datavines-engine-local-api datavines-engine-local-config datavines-engine-local-executor - datavines-engine-local-core - datavines-engine-local-transform - datavines-engine-local-connector-jdbc - datavines-engine-local-connector-file - \ No newline at end of file + diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/pom.xml deleted file mode 100644 index 5357a13f0..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/pom.xml +++ /dev/null @@ -1,134 +0,0 @@ - - - - - - datavines-engine-spark - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - jar - - datavines-engine-spark-api - - - 2.11 - 2.4.0 - 2.9.0 - provided - - - - - - org.apache.spark - spark-streaming_${scala.binary.version} - ${scope} - - - - org.apache.spark - spark-core_${scala.binary.version} - ${spark.version} - ${scope} - - - jackson-module-scala_2.11 - com.fasterxml.jackson.module - - - - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - ${scope} - - - jackson-core - com.fasterxml.jackson.core - - - - - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark.version} - - - commons-httpclient - commons-httpclient - - - org.apache.httpcomponents - httpclient - - - jackson-core-asl - org.codehaus.jackson - - - jackson-mapper-asl - org.codehaus.jackson - - - ${scope} - - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - ${scope} - - - jackson-core - com.fasterxml.jackson.core - - - - - - com.fasterxml.jackson.module - jackson-module-scala_2.11 - ${jackson.version} - ${scope} - - - jackson-core - com.fasterxml.jackson.core - - - - - - io.datavines - datavines-engine-api - ${project.version} - - - - - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment deleted file mode 100644 index d4969b187..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/resources/META-INF/services/io.datavines.engine.api.env.RuntimeEnvironment +++ /dev/null @@ -1 +0,0 @@ -io.datavines.engine.spark.api.SparkRuntimeEnvironment diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/pom.xml index cf3b9ce97..6468cd967 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-config/pom.xml @@ -39,9 +39,9 @@ io.datavines - datavines-engine-common + datavines-engine-core ${project.version} - \ No newline at end of file + diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-file/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-file/pom.xml deleted file mode 100644 index e397b1bf1..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-file/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - datavines-engine-spark - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - - datavines-engine-spark-connector-file - - - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/pom.xml deleted file mode 100644 index b35849650..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/pom.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - - datavines-engine-spark - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - jar - datavines-engine-spark-connector-jdbc - - - - io.datavines - datavines-engine-spark-api - - - - org.apache.spark - spark-streaming_${scala.binary.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - - org.apache.spark - spark-sql_${scala.binary.version} - - - - org.apache.spark - spark-hive_${scala.binary.version} - - - - io.datavines - datavines-engine-common - ${project.version} - - - - - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component deleted file mode 100644 index ddb4119a6..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component +++ /dev/null @@ -1,3 +0,0 @@ -io.datavines.engine.spark.jdbc.sink.JdbcSink -io.datavines.engine.spark.jdbc.sink.MySQLSink -io.datavines.engine.spark.jdbc.source.JdbcSource diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/pom.xml deleted file mode 100644 index 3ffba629a..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - datavines-engine-spark - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - jar - datavines-engine-spark-connector-mongodb - - - - io.datavines - datavines-engine-spark-api - - - - org.apache.spark - spark-streaming_${scala.binary.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - - org.apache.spark - spark-sql_${scala.binary.version} - - - - org.mongodb.spark - mongo-spark-connector_${scala.binary.version} - - - - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component deleted file mode 100644 index 10483d987..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component +++ /dev/null @@ -1,2 +0,0 @@ -io.datavines.engine.spark.jdbc.sink.MongodbSink -io.datavines.engine.spark.jdbc.source.MongodbSource diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/pom.xml index 37bd8fc00..ecba725c8 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/pom.xml @@ -36,5 +36,95 @@ datavines-engine-core ${project.version} + + + org.apache.spark + spark-streaming_${scala.binary.version} + ${spark.scope} + + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + ${spark.scope} + + + jackson-module-scala_2.11 + com.fasterxml.jackson.module + + + + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + ${spark.scope} + + + jackson-core + com.fasterxml.jackson.core + + + + + + org.apache.spark + spark-hive_${scala.binary.version} + ${spark.version} + ${spark.scope} + + + commons-httpclient + commons-httpclient + + + org.apache.httpcomponents + httpclient + + + jackson-core-asl + org.codehaus.jackson + + + jackson-mapper-asl + org.codehaus.jackson + + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + ${spark.scope} + + + jackson-core + com.fasterxml.jackson.core + + + + + + com.fasterxml.jackson.module + jackson-module-scala_2.11 + ${jackson.version} + ${spark.scope} + + + jackson-core + com.fasterxml.jackson.core + + + + + + org.mongodb.spark + mongo-spark-connector_${scala.binary.version} + ${spark.version} + ${spark.scope} + - \ No newline at end of file + diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/BaseSparkSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/BaseSparkSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/BaseSparkSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/BaseSparkSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/BaseSparkSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/BaseSparkSource.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/BaseSparkSource.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/BaseSparkSource.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/BaseSparkTransform.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/BaseSparkTransform.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/BaseSparkTransform.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/BaseSparkTransform.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/SparkRuntimeEnvironment.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/SparkRuntimeEnvironment.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/SparkRuntimeEnvironment.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/SparkRuntimeEnvironment.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchExecution.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchExecution.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchExecution.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchExecution.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchSource.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchSource.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/batch/SparkBatchSource.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/dialect/HiveSqlDialect.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/dialect/HiveSqlDialect.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-api/src/main/java/io/datavines/engine/spark/api/dialect/HiveSqlDialect.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/api/dialect/HiveSqlDialect.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/core/SparkDataVinesBootstrap.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/core/SparkDataVinesBootstrap.java index 4e308d720..117e06390 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/core/SparkDataVinesBootstrap.java +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/core/SparkDataVinesBootstrap.java @@ -16,9 +16,28 @@ */ package io.datavines.engine.spark.core; +import io.datavines.common.config.Config; +import io.datavines.common.config.DataVinesJobConfig; +import io.datavines.engine.api.component.Component; +import io.datavines.engine.api.env.RuntimeEnvironment; import io.datavines.engine.core.BaseDataVinesBootstrap; +import io.datavines.engine.core.enums.ConnectorType; +import io.datavines.engine.spark.api.SparkRuntimeEnvironment; +import io.datavines.engine.spark.api.batch.SparkBatchSink; +import io.datavines.engine.spark.api.batch.SparkBatchSource; +import io.datavines.engine.spark.jdbc.sink.JdbcSink; +import io.datavines.engine.spark.jdbc.sink.MongodbSink; +import io.datavines.engine.spark.jdbc.sink.MySQLSink; +import io.datavines.engine.spark.jdbc.source.JdbcSource; +import io.datavines.engine.spark.jdbc.source.MongodbSource; +import io.datavines.engine.spark.transform.sql.SqlTransform; +import java.util.List; import java.util.Base64; +import java.util.stream.Collectors; + +import static io.datavines.engine.api.EngineConstants.PLUGIN_TYPE; +import static io.datavines.engine.api.EngineConstants.TYPE; public class SparkDataVinesBootstrap extends BaseDataVinesBootstrap { @@ -30,4 +49,67 @@ public static void main(String[] args) { bootstrap.execute(args); } } + + @Override + protected RuntimeEnvironment createRuntimeEnvironment(DataVinesJobConfig config) { + SparkRuntimeEnvironment runtimeEnvironment = new SparkRuntimeEnvironment(); + Config runtimeConfig = new Config(config.getEnvConfig().getConfig()); + runtimeConfig.put(TYPE, config.getEnvConfig().getType()); + runtimeEnvironment.setConfig(runtimeConfig); + runtimeEnvironment.prepare(); + return runtimeEnvironment; + } + + @Override + protected List createSources(DataVinesJobConfig config) { + return config.getSourceParameters().stream().map(sourceConfig -> { + SparkBatchSource source; + switch (ConnectorType.of(sourceConfig.getPlugin())) { + case JDBC: + source = new JdbcSource(); + break; + case MONGODB: + source = new MongodbSource(); + break; + default: + throw new IllegalArgumentException("Unsupported spark source plugin: " + sourceConfig.getPlugin()); + } + sourceConfig.getConfig().put(PLUGIN_TYPE, sourceConfig.getType()); + source.setConfig(new Config(sourceConfig.getConfig())); + return (Component) source; + }).collect(Collectors.toList()); + } + + @Override + protected List createTransforms(DataVinesJobConfig config) { + return config.getTransformParameters().stream().map(transformConfig -> { + SqlTransform transform = new SqlTransform(); + transformConfig.getConfig().put(PLUGIN_TYPE, transformConfig.getType()); + transform.setConfig(new Config(transformConfig.getConfig())); + return (Component) transform; + }).collect(Collectors.toList()); + } + + @Override + protected List createSinks(DataVinesJobConfig config) { + return config.getSinkParameters().stream().map(sinkConfig -> { + SparkBatchSink sink; + switch (ConnectorType.of(sinkConfig.getPlugin())) { + case JDBC: + sink = new JdbcSink(); + break; + case MYSQL: + sink = new MySQLSink(); + break; + case MONGODB: + sink = new MongodbSink(); + break; + default: + throw new IllegalArgumentException("Unsupported spark sink plugin: " + sinkConfig.getPlugin()); + } + sinkConfig.getConfig().put(PLUGIN_TYPE, sinkConfig.getType()); + sink.setConfig(new Config(sinkConfig.getConfig())); + return (Component) sink; + }).collect(Collectors.toList()); + } } diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/JdbcSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/jdbc/sink/JdbcSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/JdbcSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/jdbc/sink/JdbcSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/sink/MongodbSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/jdbc/sink/MongodbSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/sink/MongodbSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/jdbc/sink/MongodbSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/MySQLSink.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/jdbc/sink/MySQLSink.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/sink/MySQLSink.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/jdbc/sink/MySQLSink.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/source/JdbcSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/jdbc/source/JdbcSource.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-jdbc/src/main/java/io/datavines/engine/spark/jdbc/source/JdbcSource.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/jdbc/source/JdbcSource.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/source/MongodbSource.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/jdbc/source/MongodbSource.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-connector-mongodb/src/main/java/io/datavines/engine/spark/jdbc/source/MongodbSource.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/jdbc/source/MongodbSource.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/java/io/datavines/engine/spark/transform/sql/SqlTransform.java b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/transform/sql/SqlTransform.java similarity index 100% rename from datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/java/io/datavines/engine/spark/transform/sql/SqlTransform.java rename to datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-core/src/main/java/io/datavines/engine/spark/transform/sql/SqlTransform.java diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/pom.xml index 6a375563c..677f1dc7a 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-executor/pom.xml @@ -31,47 +31,17 @@ datavines-engine-spark-executor - - - io.datavines - datavines-engine-api - - io.datavines datavines-engine-executor - - - - io.datavines - datavines-engine-spark-core ${project.version} io.datavines - datavines-engine-spark-connector-jdbc - ${project.version} - - - - io.datavines - datavines-engine-spark-connector-mongodb - ${project.version} - - - - io.datavines - datavines-engine-spark-connector-file - ${project.version} - - - - io.datavines - datavines-engine-spark-transform-sql + datavines-engine-spark-core ${project.version} - - \ No newline at end of file + diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/pom.xml deleted file mode 100644 index 36a26b9bc..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - datavines-engine-spark - io.datavines - 1.0.0-SNAPSHOT - - 4.0.0 - - datavines-engine-spark-transform-sql - - - - io.datavines - datavines-engine-spark-api - - - - org.apache.spark - spark-streaming_${scala.binary.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - - org.apache.spark - spark-sql_${scala.binary.version} - - - - org.apache.spark - spark-hive_${scala.binary.version} - - - - \ No newline at end of file diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component deleted file mode 100644 index 820017a22..000000000 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/datavines-engine-spark-transform-sql/src/main/resources/META-INF/services/io.datavines.engine.api.component.Component +++ /dev/null @@ -1 +0,0 @@ -io.datavines.engine.spark.transform.sql.SqlTransform diff --git a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/pom.xml b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/pom.xml index 5877e34ad..3364f9913 100644 --- a/datavines-engine/datavines-engine-plugins/datavines-engine-spark/pom.xml +++ b/datavines-engine/datavines-engine-plugins/datavines-engine-spark/pom.xml @@ -32,13 +32,8 @@ pom datavines-engine-spark-core - datavines-engine-spark-connector-jdbc datavines-engine-spark-executor - datavines-engine-spark-api - datavines-engine-spark-transform-sql datavines-engine-spark-config - datavines-engine-spark-connector-file - datavines-engine-spark-connector-mongodb @@ -117,4 +112,4 @@ - \ No newline at end of file + diff --git a/datavines-engine/pom.xml b/datavines-engine/pom.xml index 911f9e76a..851341de5 100644 --- a/datavines-engine/pom.xml +++ b/datavines-engine/pom.xml @@ -31,12 +31,10 @@ datavines-engine pom - datavines-engine-api datavines-engine-plugins datavines-engine-core datavines-engine-executor datavines-engine-config - datavines-engine-common @@ -47,4 +45,4 @@ - \ No newline at end of file + diff --git a/datavines-runner/pom.xml b/datavines-runner/pom.xml index e249a8fa7..55ec95082 100644 --- a/datavines-runner/pom.xml +++ b/datavines-runner/pom.xml @@ -59,12 +59,6 @@ ${project.version} - - io.datavines - datavines-engine-executor - ${project.version} - - io.datavines datavines-engine-spark-executor diff --git a/datavines-server/pom.xml b/datavines-server/pom.xml index fcfaa7ea3..b05280921 100644 --- a/datavines-server/pom.xml +++ b/datavines-server/pom.xml @@ -296,17 +296,11 @@ ${project.version} - - io.datavines - datavines-engine-executor - ${project.version} - - - - io.datavines - datavines-engine-spark-executor - ${project.version} - + + io.datavines + datavines-engine-spark-executor + ${project.version} + io.datavines @@ -326,12 +320,12 @@ ${project.version} - - io.datavines - datavines-engine-local-executor - ${project.version} - - + + io.datavines + datavines-engine-local-executor + ${project.version} + + slf4j-log4j12 org.slf4j @@ -348,12 +342,12 @@ ${project.version} - - io.datavines - datavines-engine-flink-executor - ${project.version} - - + + io.datavines + datavines-engine-flink-executor + ${project.version} + + org.apache.hadoop hadoop-common @@ -372,15 +366,9 @@ - - io.datavines - datavines-engine-flink-core - ${project.version} - - - - io.datavines - datavines-connector-all + + io.datavines + datavines-connector-all ${project.version} diff --git a/pom.xml b/pom.xml index 2821a9e59..2c08a9f06 100644 --- a/pom.xml +++ b/pom.xml @@ -210,19 +210,19 @@ io.datavines - datavines-engine-api + datavines-engine-core ${project.version} io.datavines - datavines-engine-core + datavines-engine-executor ${project.version} io.datavines - datavines-engine-executor + datavines-engine-local-executor ${project.version} @@ -234,7 +234,7 @@ io.datavines - datavines-engine-spark-api + datavines-engine-spark-executor ${project.version} @@ -246,7 +246,7 @@ io.datavines - datavines-engine-flink-api + datavines-engine-flink-executor ${project.version}