Merge pull request #34 from data-catering/upgrade-0.14

Upgrade 0.14

Author: pflooky

.gitignore

@@ -31,6 +31,7 @@ docker/data/custom/generated
 docker/data/custom/recordTracking
 docker/data/custom/report
 docker/sample
+docker/tmp
 
 benchmark/jars
 

Dockerfile

@@ -1,3 +1,3 @@
-FROM datacatering/data-caterer:0.13.1
+FROM datacatering/data-caterer:0.14.2
 
 COPY --chown=app:app build/libs/data-caterer-example-0.1.0.jar /opt/app/job.jar

benchmark/run_benchmark.sh

@@ -18,7 +18,7 @@ case "${uname_out}" in
     *)        sed_option="-r";;
 esac
 data_sizes=(10000 100000 1000000)
-job_names=("BenchmarkForeignKeyPlanRun" "BenchmarkJsonPlanRun" "BenchmarkParquetPlanRun") #"BenchmarkAdvancedKafkaPlanRun"
+job_names=("BenchmarkForeignKeyPlanRun" "BenchmarkJsonPlanRun" "BenchmarkParquetPlanRun")
 
 spark_query_execution_engines=("default" "blaze" "comet" "gluten")
 gluten_spark_conf="--conf \"spark.plugins=io.glutenproject.GlutenPlugin\" --conf \"spark.memory.offHeap.enabled=true\" --conf \"spark.memory.offHeap.size=1024mb\" --conf \"spark.shuffle.manager=org.apache.spark.shuffle.sort.ColumnarShuffleManager\""

docker/data/custom/task/http/http-account-task-simple.yaml

@@ -0,0 +1,71 @@
+name: "simple_http"
+steps:
+  - name: "account"
+    count:
+      records: 50
+    fields:
+      - name: "httpUrl"
+        fields:
+          - name: "url"
+            static: "http://localhost:80/anything/{id}"
+          - name: "method"
+            static: "PUT"
+          - name: "pathParam"
+            fields:
+              - name: "id"
+                options:
+                  sql: "body.account_id"
+          - name: "queryParam"
+            fields:
+              - name: "limit"
+                type: "integer"
+                options:
+                  min: 1
+                  max: 10
+      - name: "httpHeaders"
+        fields:
+          - name: "Content-Type"
+            static: "application/json"
+          - name: "Content-Length"
+          - name: "X-Account-Id"
+            options:
+              sql: "body.account_id"
+          - name: "X-Updated"
+            type: "timestamp"
+            options:
+              sql: "body.details.updated_by.time"
+      - name: "httpBody"
+        fields:
+          - name: "account_id"
+            options:
+              regex: "ACC[0-9]{8}"
+          - name: "year"
+            type: "int"
+            options:
+              min: 2021
+              max: 2022
+          - name: "amount"
+            type: "double"
+            options:
+              min: 10.0
+              max: 100.0
+          - name: "details"
+            fields:
+              - name: "name"
+              - name: "txn_date"
+                type: "date"
+                options:
+                  min: "2021-01-01"
+                  max: "2021-12-31"
+              - name: "updated_by"
+                fields:
+                  - name: "user"
+                  - name: "time"
+                    type: "timestamp"
+          - name: "transactions"
+            type: "array"
+            fields:
+              - name: "txn_date"
+                type: "date"
+              - name: "amount"
+                type: "double"

docker/data/custom/task/kafka/kafka-account-task-simple.yaml

@@ -0,0 +1,53 @@
+name: "simple_kafka"
+steps:
+  - name: "kafka_account"
+    type: "json"
+    count:
+      records: "10"
+    options:
+      topic: "account-topic"
+    fields:
+      - name: "key"
+        type: "string"
+        options:
+          sql: "body.account_id"
+      - name: "messageBody"
+        fields:
+          - name: "account_id"
+          - name: "year"
+            type: "int"
+            options:
+              min: "2021"
+              max: "2022"
+          - name: "amount"
+            type: "double"
+            options:
+              min: "10.0"
+              max: "100.0"
+          - name: "details"
+            fields:
+              - name: "name"
+              - name: "first_txn_date"
+                type: "date"
+                options:
+                  sql: "ELEMENT_AT(SORT_ARRAY(body.transactions.txn_date), 1)"
+              - name: "updated_by"
+                fields:
+                  - name: "user"
+                  - name: "time"
+                    type: "timestamp"
+          - name: "transactions"
+            type: "array"
+            fields:
+              - name: "txn_date"
+                type: "date"
+              - name: "amount"
+                type: "double"
+      - name: "messageHeaders"
+        fields:
+          - name: "account-id"
+            options:
+              sql: "body.account_id"
+          - name: "updated"
+            options:
+              sql: "body.details.update_by.time"

docker/data/custom/validation/simple-validation.yaml

@@ -7,6 +7,20 @@ dataSources:
         path: "app/src/test/resources/sample/json/txn-gen"
       validations:
         - expr: "amount < 100"
+        - field: amount
+          validation:
+            - type: "null"
+              negate: true
+            - type: "between"
+              min: 1
+              max: 10
+              description: "hello"
+              errorThreshold: 2
+            - type: "quantileValuesBetween"
+              quantileRanges:
+                0.1:
+                  - - 1.0
+                    - 10.0
         - expr: "year == 2021"
           errorThreshold: 0.1
         - expr: "regexp_like(name, 'Peter .*')"

gradle.properties

@@ -8,5 +8,5 @@ version=0.1.0
 
 scalaVersion=2.12
 scalaSpecificVersion=2.12.19
-dataCatererVersion=0.13.1
+dataCatererVersion=0.14.2
 sparkMajorVersion=3.5

helm/data-caterer/values.yaml

@@ -8,7 +8,7 @@ image:
   repository: "datacatering/data-caterer"
   pullPolicy: "IfNotPresent"
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "0.13.1"
+  tag: "0.14.2"
 
 imagePullSecrets: []
 nameOverride: ""

run.sh

@@ -1,6 +1,27 @@
 #!/usr/bin/env bash
 
+DATA_CATERER_ENV_FILE="$HOME/.data-caterer-env"
+
 data_caterer_version=$(grep dataCatererVersion gradle.properties | cut -d= -f2)
+data_caterer_user=${DATA_CATERER_API_USER:-}
+data_caterer_token=${DATA_CATERER_API_TOKEN:-}
+
+echo "Checking for Data Caterer user and token..."
+if [[ -f "$DATA_CATERER_ENV_FILE" ]]; then
+  source "$DATA_CATERER_ENV_FILE"
+else
+  if [[ -z ${DATA_CATERER_API_USER} ]]; then
+    read -p "Data Caterer user: " data_caterer_user
+    echo "export DATA_CATERER_API_USER=$data_caterer_user" > "$DATA_CATERER_ENV_FILE"
+  fi
+  if [[ -z ${DATA_CATERER_API_TOKEN} ]]; then
+    read -p "Data Caterer token: " -s data_caterer_token
+    echo "export DATA_CATERER_API_TOKEN=$data_caterer_token" >> "$DATA_CATERER_ENV_FILE"
+    echo
+  fi
+fi
+source "$DATA_CATERER_ENV_FILE"
+
 if [[ -s ".tmp_prev_class_name" ]]; then
   prev_class_name=$(cat .tmp_prev_class_name)
 else
@@ -38,12 +59,15 @@ DOCKER_CMD=(
   -v "$(pwd)/docker/sample/tracking:/opt/app/record-tracking"
   -v "$(pwd)/docker/mount:/opt/app/mount"
   -v "$(pwd)/docker/data/custom:/opt/app/custom"
+  -v "$(pwd)/docker/tmp:/tmp"
   -e "APPLICATION_CONFIG_PATH=/opt/app/custom/application.conf"
   -e "$full_class_name"
   -e "DEPLOY_MODE=client"
   -e "DRIVER_MEMORY=2g"
   -e "EXECUTOR_MEMORY=2g"
-  --network "docker_default"
+  -e "DATA_CATERER_API_USER=$DATA_CATERER_API_USER"
+  -e "DATA_CATERER_API_TOKEN=$DATA_CATERER_API_TOKEN"
+  --network "insta-infra_default"
   datacatering/data-caterer:"$data_caterer_version"
 )
 

src/main/java/io/github/datacatering/plan/AdvancedBatchEventJavaPlanRun.java

@@ -7,7 +7,7 @@
 
 public class AdvancedBatchEventJavaPlanRun extends PlanRun {
     {
-        var kafkaTask = new AdvancedKafkaJavaPlanRun().getKafkaTask();
+        var kafkaTask = new KafkaJavaPlanRun().getKafkaTask();
 
         var csvTask = csv("my_csv", "/opt/app/data/csv/account")
                 .fields(