Skip to content
This repository was archived by the owner on Aug 8, 2025. It is now read-only.

Commit 82607d5

Browse files
pflookypflooky
committed
Update to 0.13.1
1 parent 737a7cd commit 82607d5

68 files changed

Lines changed: 805 additions & 859 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docker/data/custom/plan/cassandra.yaml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ tasks:
77

88
sinkOptions:
99
foreignKeys:
10-
- - "cassandra.accounts.account_id"
11-
- - "cassandra.account_status_history.account_id"
12-
- []
10+
- source:
11+
dataSource: "cassandra"
12+
step: "accounts"
13+
fields: [ "account_id" ]
14+
generate:
15+
- dataSource: "cassandra"
16+
step: "account_status_history"
17+
fields: [ "account_id" ]

‎docker/data/custom/plan/foreign-key-example.yaml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ tasks:
1010

1111
sinkOptions:
1212
foreignKeys:
13-
- - "postgresCustomer.accounts.account_number"
14-
- - "json.account.account_id"
15-
- []
13+
- source:
14+
dataSource: "postgresCustomer"
15+
step: "accounts"
16+
fields: [ "account_number" ]
17+
generate:
18+
- dataSource: "json"
19+
step: "account"
20+
fields: [ "account_id" ]

‎docker/data/custom/plan/foreign-key.yaml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ tasks:
1010

1111
sinkOptions:
1212
foreignKeys:
13-
- - "postgresCustomer.accounts.account_number"
14-
- - "json.account.account_id"
15-
- []
13+
- source:
14+
dataSource: "postgresCustomer"
15+
step: "accounts"
16+
fields: [ "account_number" ]
17+
generate:
18+
- dataSource: "json"
19+
step: "account"
20+
fields: [ "account_id" ]

‎docker/data/custom/plan/postgres-multiple-tables.yaml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ tasks:
77

88
sinkOptions:
99
foreignKeys:
10-
- - "postgresCustomer.balances.account_number"
11-
- - "postgresCustomer.transactions.account_number"
12-
- []
10+
- source:
11+
dataSource: "postgresCustomer"
12+
step: "balances"
13+
fields: [ "account_number" ]
14+
generate:
15+
- dataSource: "postgresCustomer"
16+
step: "transactions"
17+
fields: [ "account_number" ]

‎docker/data/custom/plan/scenario-based-parquet.yaml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ tasks:
1010

1111
sinkOptions:
1212
foreignKeys:
13-
- - "postgres.accounts.account_number"
14-
- - "parquet.transactions.account_id"
15-
- []
13+
- source:
14+
dataSource: "postgres"
15+
step: "accounts"
16+
fields: [ "account_number" ]
17+
generate:
18+
- dataSource: "parquet"
19+
step: "transactions"
20+
fields: [ "account_id" ]

‎docker/data/custom/plan/scenario-based.yaml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ tasks:
1010

1111
sinkOptions:
1212
foreignKeys:
13-
- - "postgresCustomer.accounts.account_number"
14-
- - "csv.transactions.account_id"
15-
- []
13+
- source:
14+
dataSource: "postgresCustomer"
15+
step: "accounts"
16+
fields: [ "account_number" ]
17+
generate:
18+
- dataSource: "csv"
19+
step: "transactions"
20+
fields: [ "account_id" ]

‎docker/data/custom/task/cassandra/cassandra-customer-task.yaml‎

Lines changed: 48 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -7,92 +7,59 @@ steps:
77
options:
88
keyspace: "account"
99
table: "accounts"
10-
schema:
11-
fields:
12-
- name: "account_id"
13-
type: "string"
14-
generator:
15-
type: "regex"
16-
options:
17-
regex: "ACC1[0-9]{5,10}"
18-
- name: "amount"
19-
type: "double"
20-
generator:
21-
type: "random"
22-
options:
23-
min: 10.0
24-
max: 100.0
25-
- name: "name"
26-
type: "string"
27-
generator:
28-
type: "random"
29-
options:
30-
expression: "#{Name.name}"
31-
- name: "created_by"
32-
type: "string"
33-
generator:
34-
type: "random"
35-
options:
36-
expression: "#{Name.username}"
37-
- name: "status"
38-
type: "string"
39-
generator:
40-
type: "oneOf"
41-
options:
42-
oneOf:
43-
- "open"
44-
- "closed"
45-
- name: "open_time"
46-
type: "timestamp"
47-
generator:
48-
type: "random"
10+
fields:
11+
- name: "account_id"
12+
options:
13+
regex: "ACC1[0-9]{5,10}"
14+
- name: "amount"
15+
type: "double"
16+
options:
17+
min: 10.0
18+
max: 100.0
19+
- name: "name"
20+
options:
21+
expression: "#{Name.name}"
22+
- name: "created_by"
23+
options:
24+
expression: "#{Name.username}"
25+
- name: "status"
26+
options:
27+
oneOf:
28+
- "open"
29+
- "closed"
30+
- name: "open_time"
31+
type: "timestamp"
4932
- name: "account_status_history"
5033
type: "cassandra"
5134
count:
5235
records: 10
53-
perColumn:
54-
columnNames:
36+
perField:
37+
fieldNames:
5538
- "account_id"
56-
generator:
57-
type: "random"
58-
options:
59-
max: 5
60-
min: 1
39+
options:
40+
max: 5
41+
min: 1
6142
options:
6243
keyspace: "account"
6344
table: "account_status_history"
64-
schema:
65-
fields:
66-
- name: "account_id"
67-
type: "string"
68-
generator:
69-
type: "regex"
70-
options:
71-
regex: "ACC1[0-9]{5,10}"
72-
- name: "eod_date"
73-
type: "date"
74-
generator:
75-
type: "sql"
76-
options:
77-
sql: "TO_DATE(updated_time)"
78-
- name: "status"
79-
type: "string"
80-
generator:
81-
type: "oneOf"
82-
options:
83-
oneOf:
84-
- "open"
85-
- "closed"
86-
- "pending"
87-
- "active"
88-
- "disabled"
89-
- name: "updated_by"
90-
type: "string"
91-
generator:
92-
type: "random"
93-
options:
94-
expression: "#{Name.username}"
95-
- name: "updated_time"
96-
type: "timestamp"
97-
generator:
98-
type: "random"
45+
fields:
46+
- name: "account_id"
47+
options:
48+
regex: "ACC1[0-9]{5,10}"
49+
- name: "eod_date"
50+
type: "date"
51+
options:
52+
sql: "TO_DATE(updated_time)"
53+
- name: "status"
54+
options:
55+
oneOf:
56+
- "open"
57+
- "closed"
58+
- "pending"
59+
- "active"
60+
- "disabled"
61+
- name: "updated_by"
62+
options:
63+
expression: "#{Name.username}"
64+
- name: "updated_time"
65+
type: "timestamp"

‎docker/data/custom/task/file/csv/csv-transaction-task.yaml‎

Lines changed: 27 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -6,47 +6,32 @@ steps:
66
path: "/opt/app/custom/csv/transactions"
77
count:
88
records: 100
9-
perColumn:
10-
columnNames:
9+
perField:
10+
fieldNames:
1111
- "account_id"
1212
- "name"
13-
generator:
14-
type: "random"
15-
options:
16-
max: 10
17-
min: 1
18-
schema:
19-
fields:
20-
- name: "account_id"
21-
type: "string"
22-
generator:
23-
type: "regex"
24-
options:
25-
regex: "ACC1[0-9]{9}"
26-
- name: "name"
27-
type: "string"
28-
generator:
29-
type: "random"
30-
options:
31-
expression: "#{Name.name}"
32-
- name: "year"
33-
type: "int"
34-
generator:
35-
type: "random"
36-
options:
37-
min: 2021
38-
max: 2022
39-
- name: "amount"
40-
type: "double"
41-
generator:
42-
type: "random"
43-
options:
44-
min: 10.0
45-
max: 100.0
46-
- name: "txn_date"
47-
type: "date"
48-
generator:
49-
type: "random"
50-
options:
51-
min: "2021-01-01"
52-
max: "2021-12-31"
13+
options:
14+
max: 10
15+
min: 1
16+
fields:
17+
- name: "account_id"
18+
options:
19+
regex: "ACC1[0-9]{9}"
20+
- name: "name"
21+
options:
22+
expression: "#{Name.name}"
23+
- name: "year"
24+
type: "int"
25+
options:
26+
min: 2021
27+
max: 2022
28+
- name: "amount"
29+
type: "double"
30+
options:
31+
min: 10.0
32+
max: 100.0
33+
- name: "txn_date"
34+
type: "date"
35+
options:
36+
min: "2021-01-01"
37+
max: "2021-12-31"

0 commit comments

Comments
 (0)