diff --git a/data-tool/flows/common/colin_queries.py b/data-tool/flows/common/colin_queries.py index 221b2280a1..f2426e9f5b 100644 --- a/data-tool/flows/common/colin_queries.py +++ b/data-tool/flows/common/colin_queries.py @@ -76,13 +76,15 @@ def get_identifiers_per_batch(mig_batch_id: int) -> str: WHERE mcb.mig_batch_id IN ({mig_batch_id}) """ -def unfreeze_identifiers() -> str: +def unfreeze_identifiers(target_schema: str | None = 'public') -> str: + safe_schema = (target_schema or 'public').strip() or 'public' + safe_schema = '"' + safe_schema.replace('"', '""') + '"' return f""" - UPDATE corporation AS c + UPDATE {safe_schema}.corporation AS c SET corp_frozen_type_cd = NULL - FROM mig_group AS mg - JOIN mig_batch AS mb ON mb.mig_group_id = mg.id - JOIN mig_corp_batch AS mcb ON mcb.mig_batch_id = mb.id + FROM {safe_schema}.mig_group AS mg + JOIN {safe_schema}.mig_batch AS mb ON mb.mig_group_id = mg.id + JOIN {safe_schema}.mig_corp_batch AS mcb ON mcb.mig_batch_id = mb.id WHERE c.corp_num = mcb.corp_num -- cprd and mg.name in ('group_0', 'group_1', 'group_3', 'group_4','gcp_migration_group_test','misc_group') diff --git a/data-tool/flows/common/query_utils.py b/data-tool/flows/common/query_utils.py index b49bc440db..ef955c6356 100644 --- a/data-tool/flows/common/query_utils.py +++ b/data-tool/flows/common/query_utils.py @@ -109,7 +109,9 @@ def prune_candidates_from_account(pruning_corps_list: list) -> str: WHERE corp_num IN {in_list} """ -def get_cutoff_timestamp_query() -> str: +def get_cutoff_timestamp_query(target_schema: str | None = 'public') -> str: + safe_schema = (target_schema or 'public').strip() or 'public' + safe_schema = '"' + safe_schema.replace('"', '""') + '"' return f""" - SELECT extracted_at FROM colin_extract_version + SELECT extracted_at FROM {safe_schema}.colin_extract_version """ diff --git a/data-tool/flows/refresh_extract_subset_flow.py b/data-tool/flows/refresh_extract_subset_flow.py index c3eedc8d6b..5854d4bad9 100644 --- a/data-tool/flows/refresh_extract_subset_flow.py +++ b/data-tool/flows/refresh_extract_subset_flow.py @@ -1,6 +1,8 @@ import argparse import os +from dataclasses import dataclass from pathlib import Path +from enum import Enum import re import subprocess import sys @@ -26,6 +28,84 @@ _BUILD_VIEWS_SCRIPT = _REPO_ROOT / 'data-tool' / 'scripts' / 'colin_corps_extract_postgres_views_ddl' +# ========================= +# cfg_* (types & config) +# ========================= + +class cfg_GenerationMode(str, Enum): + REFRESH = "refresh" # delete + reload + LOAD = "load" # load only + + +class cfg_RenderMode(str, Enum): + INLINE = "inline" # render templates into chunk files (no vset) + VSET = "vset" # legacy behavior (runtime vset substitution) + + +class cfg_OracleInStrategy(str, Enum): + AUTO = "auto" + CHUNK_FILES = "chunk_files" + OR_OF_IN_LISTS = "or_of_in_lists" + + +class cfg_PgDisableMethod(str, Enum): + TABLE_TRIGGERS = "table_triggers" # ALTER TABLE ... DISABLE/ENABLE TRIGGER ALL (default) + REPLICA_ROLE = "replica_role" # SET session_replication_role=replica|origin (superuser only) + + +class cfg_DeltaScope(str, Enum): + BATCH = "batch" # only process identifiers in the current batch + FULL = "full" # process all identifiers in the corp list + +@dataclass +class SubsetConfig: + corp_file: Path + mode: cfg_GenerationMode + chunk_size: int + threads: int + prefix_numeric_bc: bool + include_cp: bool + + pg_fastload: bool + pg_disable_method: cfg_PgDisableMethod + + out_master: Path + run_dbschemacli: bool + dbschemacli_cmd: str + + refresh_views: bool + + source_connection: str + target_connection: str + target_schema: str + + reset_extract_postgres: bool = True + delta_scope: cfg_DeltaScope = cfg_DeltaScope.BATCH + + +def build_configs(args: argparse.Namespace) -> SubsetConfig: + """Build SubsetConfig from parsed arguments.""" + return SubsetConfig( + corp_file=Path(args.corp_file).expanduser().resolve(), + mode=cfg_GenerationMode(args.mode), + delta_scope=cfg_DeltaScope(args.delta_scope), + chunk_size=args.chunk_size, + threads=args.threads, + prefix_numeric_bc=(args.mode == 'refresh'), + include_cp=args.include_cp, + pg_fastload=args.pg_fastload, + pg_disable_method=cfg_PgDisableMethod(args.pg_disable_method), + out_master=_resolve_master_script_path(args.out), + run_dbschemacli=args.run_dbschemacli, + dbschemacli_cmd=args.dbschemacli_cmd, + refresh_views=args.refresh_views, + reset_extract_postgres=args.reset_extract_postgres, + source_connection=args.source_connection, + target_connection=args.target_connection, + target_schema=args.target_schema + ) + + def _resolve_master_script_path(out: str | None) -> Path: if not out: return _SUBSET.resolve() @@ -47,14 +127,25 @@ def require_file(path: str | Path, description: str) -> Path: return resolved -def _reset_extract_postgres_db() -> None: +def _normalize_target_schema(target_schema: str | None, default_schema: str = 'public') -> str: + normalized = (target_schema or default_schema).strip() + return normalized or default_schema + + +def _reset_extract_postgres_db(target_schema: str | None = 'public') -> None: cfg = get_named_config() dbname = cfg.DB_NAME_COLIN_MIGR host = cfg.DB_HOST_COLIN_MIGR port = str(cfg.DB_PORT_COLIN_MIGR) user = cfg.DB_USER_COLIN_MIGR password = cfg.DB_PASSWORD_COLIN_MIGR - + + target_schema = _normalize_target_schema(target_schema) + safe_schema = '"' + target_schema.replace('"', '""') + '"' + search_path_sql = f'SET search_path TO {safe_schema};' + create_schema_sql = f'CREATE SCHEMA IF NOT EXISTS {safe_schema};' + psql_schema_vars = ['-v', f'schema_name={target_schema}'] + require_file(_DEFAULT_DDL, 'Extract DDL File') pg_flags = ['-h', host, '-p', str(port), '-U', user] @@ -70,8 +161,37 @@ def _reset_extract_postgres_db() -> None: _run_cmd(['psql', *pg_flags, '-d', 'postgres', '-c', terminate_sql ], env=run_env) _run_cmd(['dropdb', *pg_flags, '--maintenance-db=postgres', '--if-exists', dbname ], env=run_env) _run_cmd(['createdb', *pg_flags, '--maintenance-db=postgres', '-T', 'template0', dbname ], env=run_env) - _run_cmd(['psql', *pg_flags, '-d', dbname, '-v', 'ON_ERROR_STOP=1', '-f', str(_DEFAULT_DDL) ], env=run_env) - _run_cmd(['psql', *pg_flags, '-d', dbname, '-v', 'ON_ERROR_STOP=1', '-f', str(_BUILD_VIEWS_SCRIPT) ], env=run_env) + _run_cmd(['psql', *pg_flags, '-d', dbname, '-v', 'ON_ERROR_STOP=1', *psql_schema_vars, '-c', create_schema_sql], env=run_env) + _run_cmd(['psql', *pg_flags, '-d', dbname, '-v', 'ON_ERROR_STOP=1', *psql_schema_vars, '-c', search_path_sql, '-f', str(_DEFAULT_DDL)], env=run_env) + _run_cmd(['psql', *pg_flags, '-d', dbname, '-v', 'ON_ERROR_STOP=1', *psql_schema_vars, '-c', search_path_sql, '-f', str(_BUILD_VIEWS_SCRIPT)], env=run_env) + + + +# def _reset_extract_postgres_db() -> None: +# cfg = get_named_config() +# dbname = cfg.DB_NAME_COLIN_MIGR +# host = cfg.DB_HOST_COLIN_MIGR +# port = str(cfg.DB_PORT_COLIN_MIGR) +# user = cfg.DB_USER_COLIN_MIGR +# password = cfg.DB_PASSWORD_COLIN_MIGR + +# require_file(_DEFAULT_DDL, 'Extract DDL File') + +# pg_flags = ['-h', host, '-p', str(port), '-U', user] +# run_env = dict(os.environ) +# if password and 'PGPASSWORD' not in run_env: +# run_env['PGPASSWORD'] = password +# safe_db = str(dbname).replace("'", "''") +# terminate_sql = ( +# "SELECT pg_terminate_backend(pg_stat_activity.pid) " +# "FROM pg_stat_activity " +# f"WHERE datname = '{safe_db}' AND pid <> pg_backend_pid();" +# ) +# _run_cmd(['psql', *pg_flags, '-d', 'postgres', '-c', terminate_sql ], env=run_env) +# _run_cmd(['dropdb', *pg_flags, '--maintenance-db=postgres', '--if-exists', dbname ], env=run_env) +# _run_cmd(['createdb', *pg_flags, '--maintenance-db=postgres', '-T', 'template0', dbname ], env=run_env) +# _run_cmd(['psql', *pg_flags, '-d', dbname, '-v', 'ON_ERROR_STOP=1', '-f', str(_DEFAULT_DDL) ], env=run_env) +# _run_cmd(['psql', *pg_flags, '-d', dbname, '-v', 'ON_ERROR_STOP=1', '-f', str(_BUILD_VIEWS_SCRIPT) ], env=run_env) @task(name='Get-Fallen-Out-Identifiers', cache_policy=NO_CACHE) def get_fallen_identifiers(updated_corp_nums: list) -> list[dict]: @@ -106,25 +226,28 @@ def prune_fallen_identifiers(fallenout_corp_nums: list) -> list[dict]: prune_account = conn.execute(text(account_query)) print(f"Pruned corp_processing={prune_cp.rowcount}, mig_corp_batch={prune_batch.rowcount}, mig_corp_account={prune_account.rowcount}") -def get_cuttoff_timestamp() -> datetime: +def get_cuttoff_timestamp(target_schema: str | None = 'public') -> datetime: cfg = get_named_config() - cuttoff_timestamp = get_cutoff_timestamp_query() + cuttoff_timestamp = get_cutoff_timestamp_query(target_schema) with create_engine(cfg.SQLALCHEMY_DATABASE_URI_COLIN_MIGR).begin() as conn: cuttoff_timestamp_result = conn.execute(text(cuttoff_timestamp)).scalar() print(f"cuttoff timestamp is {cuttoff_timestamp_result}") return cuttoff_timestamp_result +# @task(name='Cleanup-Extract-Postgres', cache_policy=NO_CACHE) +# def cleanup_extract_postgres_db() -> None: +# _reset_extract_postgres_db() @task(name='Cleanup-Extract-Postgres', cache_policy=NO_CACHE) -def cleanup_extract_postgres_db() -> None: - _reset_extract_postgres_db() +def cleanup_extract_postgres_db(target_schema: str | None = 'public') -> None: + _reset_extract_postgres_db(target_schema=target_schema) @task(name='Unfreeze-Identifiers', cache_policy=NO_CACHE) -def run_unfreeze_identifiers() -> None: +def run_unfreeze_identifiers(target_schema: str | None = 'public') -> None: cfg = get_named_config() with create_engine(cfg.SQLALCHEMY_DATABASE_URI_COLIN_MIGR).begin() as conn: - result = conn.execute(text(unfreeze_identifiers())) + result = conn.execute(text(unfreeze_identifiers(target_schema=target_schema))) print(f'Unfroze corporation rows={result.rowcount}') @task(name='Get-Updated-Identifiers-Colin', cache_policy=NO_CACHE) @@ -147,9 +270,51 @@ def get_updated_identifiers_colin(cutoff_timestamp: str, mig_batch_id: int, coli rows = [dict(row) for row in result.mappings()] return rows + +@task(name='Run-CPRD-Subset-Generator', cache_policy=NO_CACHE) +def run_cprd_subset_extract_generator(cfg: SubsetConfig) -> subprocess.CompletedProcess: + """ + Generate Commands + """ + require_file(_SCRIPT_PATH, 'Generated script') + corp_path =require_file(cfg.corp_file, 'Corp list file') + argv = [ + sys.executable, + str(_SCRIPT_PATH), + '--corp-file', + str(corp_path), + '--mode', + cfg.mode, + '--chunk-size', + str(cfg.chunk_size), + '--threads', + str(cfg.threads), + '--pg-disable-method', + cfg.pg_disable_method, + ] + argv.extend(['--target-connection', cfg.target_connection]) + argv.extend(['--source-connection', cfg.source_connection]) + argv.extend(['--target-schema', cfg.target_schema]) + if cfg.pg_fastload: + argv.append('--pg-fastload') + if cfg.include_cp: + argv.append('--include-cp') + if cfg.prefix_numeric_bc: + argv.append('--prefix-numeric-bc') + out_path = _resolve_master_script_path(cfg.out_master) + out_path.parent.mkdir(parents=True, exist_ok=True) + argv.extend(['--out', str(out_path)]) + return subprocess.run( + argv, + cwd=str(_REPO_ROOT), + capture_output=False, + text=True, + ) + + @task(name='Run-CPRD-Subset-Generator', cache_policy=NO_CACHE) -def run_cprd_subset_extract_generator( +def run_cprd_subset_extract_generator_original( corp_file: str, mode: str, chunk_size: int, @@ -213,9 +378,10 @@ def run_dbschemacli_task(master_script: str, dbschemacli_cmd: str = 'dbschemacli ) @task(name='Refresh-Views', cache_policy=NO_CACHE) -def run_refresh_views(mode: str = 'refresh', targets: str = 'all') -> subprocess.CompletedProcess: +def run_refresh_views(mode: str = 'refresh', targets: str = 'all', schema: str | None = 'public') -> subprocess.CompletedProcess: cfg = get_named_config() script = require_file(_REFRESH_VIEWS_SCRIPT, 'refresh_colin_extract_views.sh') + normalized_schema = _normalize_target_schema(schema) argv = [ str(script), '--mode', mode, @@ -223,7 +389,8 @@ def run_refresh_views(mode: str = 'refresh', targets: str = 'all') -> subproces '--db', cfg.DB_NAME_COLIN_MIGR, '--host', cfg.DB_HOST_COLIN_MIGR, '--port', str(cfg.DB_PORT_COLIN_MIGR), - '--user', cfg.DB_USER_COLIN_MIGR + '--user', cfg.DB_USER_COLIN_MIGR, + '--schema', normalized_schema, ] run_env = dict(os.environ) if cfg.DB_PASSWORD_COLIN_MIGR and 'PGPASSWORD' not in run_env: @@ -237,6 +404,84 @@ def run_refresh_views(mode: str = 'refresh', targets: str = 'all') -> subproces ) +@flow(name='Extract-Subset-Flow', log_prints=True, persist_result=False) +def extract_flow(cfg: SubsetConfig) -> None: + print(f'Running Extract-Subset-Flow with config: {cfg} ') + """ + Generate files + """ + if cfg.mode == 'refresh': + cfg.reset_extract_postgres = False + print('Running in refresh mode: skipping Postgres DB reset') + if cfg.reset_extract_postgres: + cleanup_extract_postgres_db(cfg.target_schema) + + cutoff = get_cuttoff_timestamp(cfg.target_schema) + + config = get_config() + colin_oracle_engine = colin_oracle_init(config) + # Get Identifiers + feed_path: Path | None = None + if cfg.mode == 'refresh': + updated_rows = get_updated_identifiers_colin(cutoff_timestamp=cutoff, + mig_batch_id=config.MIG_BATCH_IDS, + colin_oracle_engine=colin_oracle_engine, + chunk_size=cfg.chunk_size, + scope=cfg.delta_scope) + print(f'Colin updated identifiers : {len(updated_rows)} rows') + _GENERATED_DIR.mkdir(parents=True, exist_ok=True) + feed_path = _GENERATED_DIR / f'refresh_corp_feed_{os.getpid()}.tmp' + seen = set() + lines = [] + updated_corp_nums = [] + for row in updated_rows: + for k, v in row.items(): + if k is None or v is None: + continue + if str(k).lower() == 'corp_num': + c = str(v).strip() + if c and c not in seen: + seen.add(c) + lines.append(c) + updated_corp_nums.append('BC'+c) + break + if not lines: + raise ValueError('refresh: no corp_num in updated_rows') + feed_path.write_text('\n'.join(lines) + '\n', encoding='utf-8') + corp_file = str(feed_path) + result: subprocess.CompletedProcess | None = None + print(f'Running CPRD subset extract generator {cfg.corp_file}') + try: + result = run_cprd_subset_extract_generator(cfg) + finally: + if feed_path is not None: + feed_path.unlink(missing_ok=True) + if result.returncode != 0 and result is not None: + raise RuntimeError(f'Generator exited with code {result.returncode}') + print(f'generator completed successfully') + + if cfg.run_dbschemacli: + master_script = _resolve_master_script_path(out=cfg.out_master) + run_result = run_dbschemacli_task( + master_script=str(master_script), + dbschemacli_cmd=cfg.dbschemacli_cmd, + ) + if run_result.returncode != 0: + raise RuntimeError(f'DbSchemaCLI exited with code {run_result.returncode}') + + print('Running Unfreezing Corps.......') + run_unfreeze_identifiers(cfg.target_schema) + + if cfg.refresh_views and cfg.delta_scope == 'batch': + refresh_result = run_refresh_views('refresh', 'all', cfg.target_schema) + if refresh_result.returncode !=0: + raise RuntimeError(f'Refresh-Views exited with code {refresh_result.returncode}') + if cfg.mode == 'refresh' and cfg.delta_scope == 'batch': + prune_identifiers = get_fallen_identifiers(updated_corp_nums) + prune_fallen_identifiers(prune_identifiers) + + + @flow(name='Extract-Subset-Flow', log_prints=True, persist_result=False) def extract_pull_flow( corp_file: str, @@ -252,7 +497,8 @@ def extract_pull_flow( reset_extract_postgres: bool = True, include_cp: bool = False, target_connection: str = _DEFAULT_TARGET_CONNECTION, - delta_scope: str = 'batch' + delta_scope: str = 'batch', + target_schema: str = 'colin_extract', ) -> None: """ Generate files @@ -261,9 +507,10 @@ def extract_pull_flow( reset_extract_postgres = False print('Running in refresh mode: skipping Postgres DB reset') if reset_extract_postgres: - cleanup_extract_postgres_db() + # cleanup_extract_postgres_db() + cleanup_extract_postgres_db(target_schema) - cutoff = get_cuttoff_timestamp() + cutoff = get_cuttoff_timestamp(target_schema) config = get_config() colin_oracle_engine = colin_oracle_init(config) @@ -327,16 +574,16 @@ def extract_pull_flow( run_unfreeze_identifiers() if refresh_views and delta_scope == 'batch': - refresh_result = run_refresh_views('refresh', 'all') + refresh_result = run_refresh_views('refresh', 'all', target_schema) if refresh_result.returncode !=0: raise RuntimeError(f'Refresh-Views exited with code {refresh_result.returncode}') if mode == 'refresh' and delta_scope == 'batch': prune_identifiers = get_fallen_identifiers(updated_corp_nums) prune_fallen_identifiers(prune_identifiers) -if __name__ == '__main__': +def build_arg_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(description='Run Extract-Pull flow....') - p.add_argument('--corp_file', default='../data-tool/scripts/generated/delta_ctst.txt', help='Path to newline-delimited corp identifiers') + p.add_argument('--corp_file', default='../data-tool/scripts/generated/corp_ids_ctst.txt', help='Path to newline-delimited corp identifiers') p.add_argument('--mode', default='refresh', choices=('refresh', 'load')) p.add_argument('--delta-scope', default='batch', choices=('batch', 'full')) p.add_argument('--chunk-size', type=int, default=900, help='Max items per IN list.') @@ -347,7 +594,21 @@ def extract_pull_flow( p.add_argument('--out', default='data-tool/scripts/subset/generated/subset_refresh.sql', help='Output path for generated master script.') p.add_argument('--run-dbschemacli', action='store_false') p.add_argument('--refresh-views', action='store_false') - p.add_argument('--dbschemacli-cmd', default='dbschemacli') + p.add_argument('--dbschemacli-cmd', default='/usr/local/bin/DbSchemaCLI') p.add_argument('--reset-extract-postgres', action='store_false') - p.add_argument('--target-connection', default=_DEFAULT_TARGET_CONNECTION) - extract_pull_flow(**vars(p.parse_args())) + p.add_argument('--source-connection', default='ctst') + p.add_argument('--target-connection', default='ctst_pg') + p.add_argument('--target-schema', default='public') + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_arg_parser().parse_args(argv) + config = build_configs(args) + extract_flow(config) + # extract_pull_flow(config) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/data-tool/refresh_colin_extract_views.sh b/data-tool/refresh_colin_extract_views.sh index d09c8bbfd1..0320e20f06 100755 --- a/data-tool/refresh_colin_extract_views.sh +++ b/data-tool/refresh_colin_extract_views.sh @@ -461,8 +461,13 @@ while [[ $# -gt 0 ]]; do shift 2 ;; --schema) - PGSCHEMA="${2:-}" - shift 2 + if [[ $# -gt 1 && "$2" != -* ]]; then + PGSCHEMA="$2" + shift 2 + else + PGSCHEMA="${PGSCHEMA:-public}" + shift + fi ;; --psql-bin) PSQL_BIN="${2:-}" diff --git a/data-tool/scripts/_generated/subset_load.sql b/data-tool/scripts/_generated/subset_load.sql new file mode 100644 index 0000000000..20336dfe31 --- /dev/null +++ b/data-tool/scripts/_generated/subset_load.sql @@ -0,0 +1,48 @@ +vset cli.settings.ignore_errors=false +vset cli.settings.replace_variables=false +vset cli.settings.transfer_threads=4 +vset format.date=YYYY-MM-dd'T'hh:mm:ss'Z' +vset format.timestamp=YYYY-MM-dd'T'hh:mm:ss'Z' + +connect ctst_pg_subset; +-- Serialize subset runs on this target DB. +execute /home/kdeodhar/repos/lear/data-tool/scripts/subset/subset_pg_acquire_advisory_lock.sql + +-- Prepare shared address staging table before learning schema +execute /home/kdeodhar/repos/lear/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_prepare_address_stage.sql +learn schema colin_extract; + +truncate table colin_extract.colin_extract_version; insert into colin_extract.colin_extract_version (extracted_at) values (current_timestamp); + +-- Postgres fast-load mode (session-level settings) +execute /home/kdeodhar/repos/lear/data-tool/scripts/subset/subset_pg_fastload_begin.sql + +-- Postgres helper: allow VARCHAR/BPCHAR -> BOOLEAN assignment (DbSchemaCLI boolean inserts) +execute /home/kdeodhar/repos/lear/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_boolean_casts.sql +-- Fail-fast: verify varchar/bpchar -> boolean casts exist +select 't'::varchar::boolean; +select 'f'::bpchar::boolean; + +execute /home/kdeodhar/repos/lear/data-tool/scripts/_generated/subset_load_chunks/support/subset_disable_triggers.sql + +-- global cars* refresh (not corp-scoped; full dataset truncate + reload) +execute /home/kdeodhar/repos/lear/data-tool/scripts/_generated/subset_load_chunks/support/subset_delete_cars.sql +execute /home/kdeodhar/repos/lear/data-tool/scripts/_generated/subset_load_chunks/support/subset_transfer_cars.sql + +-- transfer corp-scoped subset from Oracle to Postgres +-- transfer chunk 001/001 +execute /home/kdeodhar/repos/lear/data-tool/scripts/_generated/subset_load_chunks/transfer_all.sql + +-- purge BCOMPS-excluded corps (computed in Postgres after load) +execute /home/kdeodhar/repos/lear/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_purge_bcomps_excluded.sql + +execute /home/kdeodhar/repos/lear/data-tool/scripts/_generated/subset_load_chunks/support/subset_enable_triggers.sql + +-- Cleanup shared address staging table +execute /home/kdeodhar/repos/lear/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_cleanup_address_stage.sql + +-- Release subset-run advisory lock +execute /home/kdeodhar/repos/lear/data-tool/scripts/subset/subset_pg_release_advisory_lock.sql + +-- Reset Postgres fast-load session settings +execute /home/kdeodhar/repos/lear/data-tool/scripts/subset/subset_pg_fastload_end.sql diff --git a/data-tool/scripts/_generated/subset_load_chunks/support/subset_delete_cars.sql b/data-tool/scripts/_generated/subset_load_chunks/support/subset_delete_cars.sql new file mode 100644 index 0000000000..f13b53082d --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/support/subset_delete_cars.sql @@ -0,0 +1,10 @@ +-- Global delete/clear for cars* tables (subset refresh/load). +-- Intended to be executed while connected to target Postgres extract DB (cprd_pg). +-- +-- These tables are NOT corp-scoped, so we truncate the entire dataset and reload from Oracle. +-- Volume is low enough that a full refresh is appropriate. + +TRUNCATE TABLE colin_extract.carindiv; +TRUNCATE TABLE colin_extract.carsrept; +TRUNCATE TABLE colin_extract.carsbox; +TRUNCATE TABLE colin_extract.carsfile; diff --git a/data-tool/scripts/_generated/subset_load_chunks/support/subset_delete_chunk.sql b/data-tool/scripts/_generated/subset_load_chunks/support/subset_delete_chunk.sql new file mode 100644 index 0000000000..68728542a1 --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/support/subset_delete_chunk.sql @@ -0,0 +1,119 @@ +-- Delete a chunk of corps from the TARGET Postgres extract DB. +-- +-- REQUIRED DbSchemaCLI variables (replace_variables=true): +-- corp_ids_in : comma-separated SQL string literals for target corp_num values (NO parentheses), +-- e.g. 'BC0460007','A1234567' +-- +-- Intended to be executed from a master DbSchemaCLI script connected to the target Postgres DB (colin_extract schema). +-- +-- Note: This script intentionally does NOT delete internal migration/processing tables (mig_*, corp_processing, +-- colin_tracking, affiliation_processing, etc). It only deletes the corp-scoped COLIN extract tables that are +-- reloaded from Oracle. +-- IMPORTANT: +-- - Because preserved processing/tracking tables still reference corporation/event rows, refresh mode must keep +-- FK enforcement suppressed across this delete/reload window (for example via replica_role, or by disabling +-- triggers on the preserved referencing tables too). + +-- Address rows are treated as shared/global during subset refresh. +-- Do not delete them here: subset_transfer_chunk.sql stages incoming Oracle address rows and +-- merges them into the configured target address table by addr_id. + +-- Delete child tables first (event-scoped). +DELETE FROM colin_extract.notification_resend +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.notification +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.filing_user +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.payment +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.ledger_text +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.conv_ledger +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.conv_event +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.completing_party +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.submitting_party +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.corp_involved_amalgamating +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.corp_involved_cont_in +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.correction +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.filing +WHERE event_id IN (SELECT event_id FROM colin_extract.event WHERE corp_num IN (&corp_ids_in)); + +-- Delete corp-party related tables. +DELETE FROM colin_extract.party_notification +WHERE party_id IN (SELECT corp_party_id FROM colin_extract.corp_party WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.offices_held +WHERE corp_party_id IN (SELECT corp_party_id FROM colin_extract.corp_party WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.corp_party_relationship +WHERE corp_party_id IN (SELECT corp_party_id FROM colin_extract.corp_party WHERE corp_num IN (&corp_ids_in)); + +DELETE FROM colin_extract.corp_party +WHERE corp_num IN (&corp_ids_in); + +-- Delete corp-scoped tables. +DELETE FROM colin_extract.office +WHERE corp_num IN (&corp_ids_in); + +DELETE FROM colin_extract.corp_name +WHERE corp_num IN (&corp_ids_in); + +DELETE FROM colin_extract.corp_state +WHERE corp_num IN (&corp_ids_in); + +DELETE FROM colin_extract.corp_comments +WHERE corp_num IN (&corp_ids_in); + +DELETE FROM colin_extract.corp_flag +WHERE corp_num IN (&corp_ids_in); + +DELETE FROM colin_extract.cont_out +WHERE corp_num IN (&corp_ids_in); + +DELETE FROM colin_extract.corp_restriction +WHERE corp_num IN (&corp_ids_in); + +DELETE FROM colin_extract.jurisdiction +WHERE corp_num IN (&corp_ids_in); + +DELETE FROM colin_extract.resolution +WHERE corp_num IN (&corp_ids_in); + +DELETE FROM colin_extract.share_series +WHERE corp_num IN (&corp_ids_in); + +DELETE FROM colin_extract.share_struct_cls +WHERE corp_num IN (&corp_ids_in); + +DELETE FROM colin_extract.share_struct +WHERE corp_num IN (&corp_ids_in); + +-- Delete events last (many tables reference event_id). +DELETE FROM colin_extract.event +WHERE corp_num IN (&corp_ids_in); + +-- Delete the corp rows last. +DELETE FROM colin_extract.corporation +WHERE corp_num IN (&corp_ids_in); + +-- Address rows are refreshed via stage+merge in subset_transfer_chunk.sql. diff --git a/data-tool/scripts/_generated/subset_load_chunks/support/subset_disable_triggers.sql b/data-tool/scripts/_generated/subset_load_chunks/support/subset_disable_triggers.sql new file mode 100644 index 0000000000..02de0f1b14 --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/support/subset_disable_triggers.sql @@ -0,0 +1,38 @@ +-- Disable triggers for corp-scoped tables (subset refresh/load). +-- Intended to be executed from a master DbSchemaCLI script while connected to the target Postgres DB. + +ALTER TABLE colin_extract.corporation DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_name DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_state DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.event DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.filing DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.filing_user DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.office DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_comments DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.ledger_text DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_party DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_party_relationship DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.offices_held DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.completing_party DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.submitting_party DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_flag DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.cont_out DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.conv_event DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.conv_ledger DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_involved_amalgamating DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_involved_cont_in DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_restriction DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.correction DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.jurisdiction DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.resolution DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.share_series DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.share_struct DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.share_struct_cls DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.notification DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.notification_resend DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.party_notification DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.payment DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.carsfile DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.carsbox DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.carsrept DISABLE TRIGGER ALL; +ALTER TABLE colin_extract.carindiv DISABLE TRIGGER ALL; diff --git a/data-tool/scripts/_generated/subset_load_chunks/support/subset_enable_triggers.sql b/data-tool/scripts/_generated/subset_load_chunks/support/subset_enable_triggers.sql new file mode 100644 index 0000000000..86d272effe --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/support/subset_enable_triggers.sql @@ -0,0 +1,38 @@ +-- Enable triggers for corp-scoped tables (subset refresh/load). +-- Intended to be executed from a master DbSchemaCLI script while connected to the target Postgres DB. + +ALTER TABLE colin_extract.corporation ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_name ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_state ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.event ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.filing ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.filing_user ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.office ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_comments ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.ledger_text ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_party ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_party_relationship ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.offices_held ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.completing_party ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.submitting_party ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_flag ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.cont_out ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.conv_event ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.conv_ledger ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_involved_amalgamating ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_involved_cont_in ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.corp_restriction ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.correction ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.jurisdiction ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.resolution ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.share_series ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.share_struct ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.share_struct_cls ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.notification ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.notification_resend ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.party_notification ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.payment ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.carsfile ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.carsbox ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.carsrept ENABLE TRIGGER ALL; +ALTER TABLE colin_extract.carindiv ENABLE TRIGGER ALL; diff --git a/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_boolean_casts.sql b/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_boolean_casts.sql new file mode 100644 index 0000000000..272e80b4c5 --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_boolean_casts.sql @@ -0,0 +1,55 @@ +-- Postgres helper for DbSchemaCLI subset extracts. +-- +-- Problem: +-- DbSchemaCLI transfers from Oracle to Postgres commonly bind Oracle CHAR/VARCHAR values as JDBC VARCHAR. +-- When the TARGET column is BOOLEAN, Postgres does NOT have an assignment cast from VARCHAR/BPCHAR -> BOOLEAN, +-- so inserts like: +-- ... SEND_AR_IND = 't' +-- can fail with: +-- ERROR: column "send_ar_ind" is of type boolean but expression is of type character varying +-- +-- Solution: +-- Install IMPLICIT casts for varchar/bpchar -> boolean that use Postgres' standard boolean input rules +-- ('t'/'f', 'true'/'false', '1'/'0', 'yes'/'no', etc). +-- +-- NOTE: +-- Some DbSchemaCLI builds appear to mangle the keyword "ASSIGNMENT" when executing scripts, producing +-- Postgres errors like: +-- ERROR: syntax error at or near "ASSIGNMEN" +-- Using IMPLICIT still enables automatic casting for INSERT/UPDATE (it is stronger than ASSIGNMENT), +-- while avoiding that parsing issue. +-- +-- IMPORTANT (DbSchemaCLI compatibility): +-- DbSchemaCLI splits statements on semicolons and does not reliably handle semicolons inside dollar-quoted +-- bodies. Keep dollar-quoted bodies free of internal semicolons and avoid DO $$ blocks. + +CREATE OR REPLACE FUNCTION colin_extract.dbcli_varchar_to_boolean(val varchar) +RETURNS boolean +LANGUAGE sql +IMMUTABLE +STRICT +AS $$ + SELECT (val::text)::boolean +$$; + +CREATE OR REPLACE FUNCTION colin_extract.dbcli_bpchar_to_boolean(val bpchar) +RETURNS boolean +LANGUAGE sql +IMMUTABLE +STRICT +AS $$ + SELECT (val::text)::boolean +$$; + +-- Recreate casts in an idempotent way (Postgres has no CREATE CAST IF NOT EXISTS). +DROP CAST IF EXISTS (varchar AS boolean); +CREATE CAST (varchar AS boolean) + WITH FUNCTION colin_extract.dbcli_varchar_to_boolean(varchar) + AS IMPLICIT -- DbSchemaCLI workaround: avoid keyword being last token +; + +DROP CAST IF EXISTS (bpchar AS boolean); +CREATE CAST (bpchar AS boolean) + WITH FUNCTION colin_extract.dbcli_bpchar_to_boolean(bpchar) + AS IMPLICIT -- DbSchemaCLI workaround: avoid keyword being last token +; diff --git a/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_cleanup_address_stage.sql b/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_cleanup_address_stage.sql new file mode 100644 index 0000000000..eb4fa28551 --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_cleanup_address_stage.sql @@ -0,0 +1,4 @@ +-- Cleanup the shared address staging table used by subset_transfer_chunk.sql. +-- No-op: the helper table is predeclared in the COLIN extract DDL and is truncated during prepare/chunk execution. + +TRUNCATE TABLE colin_extract.subset_address_stage; diff --git a/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_cleanup_orphan_children.sql b/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_cleanup_orphan_children.sql new file mode 100644 index 0000000000..09cb9a3a5d --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_cleanup_orphan_children.sql @@ -0,0 +1,62 @@ +-- Cleanup stale child rows that no longer have the parent rows used by refresh-mode deletes. +-- +-- Why this exists: +-- - refresh-mode chunk deletes remove event-scoped rows by first looking up event_id in target `event` +-- - and remove corp-party child rows by first looking up corp_party_id in target `corp_party` +-- - so a prior failed/interleaved run can leave stale child rows behind when the parent row is missing +-- - those orphans can then collide with the next reload (for example, unique `filing.event_id`) +-- +-- This cleanup is intentionally narrow: +-- - only rows whose normal refresh delete path traverses a parent lookup are removed here +-- - corp-scoped rows deleted directly by corp_num are left to the regular chunk deletes + +-- Event-scoped children whose parent event row is missing. +DELETE FROM colin_extract.notification_resend t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.notification t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.filing_user t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.payment t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.ledger_text t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.conv_ledger t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.conv_event t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.completing_party t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.submitting_party t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.corp_involved_amalgamating t +WHERE t.event_id IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.corp_involved_cont_in t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.correction t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +DELETE FROM colin_extract.filing t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.event e WHERE e.event_id = t.event_id); + +-- Corp-party children whose parent corp_party row is missing. +DELETE FROM colin_extract.party_notification t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.corp_party cp WHERE cp.corp_party_id = t.party_id); + +DELETE FROM colin_extract.offices_held t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.corp_party cp WHERE cp.corp_party_id = t.corp_party_id); + +DELETE FROM colin_extract.corp_party_relationship t +WHERE NOT EXISTS (SELECT 1 FROM colin_extract.corp_party cp WHERE cp.corp_party_id = t.corp_party_id); diff --git a/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_prepare_address_stage.sql b/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_prepare_address_stage.sql new file mode 100644 index 0000000000..38be25e10f --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_prepare_address_stage.sql @@ -0,0 +1,4 @@ +-- Prepare the shared address staging table used by subset_transfer_chunk.sql. +-- This is a predeclared regular table (not TEMP) because DbSchemaCLI transfer work may use separate sessions. + +TRUNCATE TABLE colin_extract.subset_address_stage; diff --git a/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_purge_bcomps_excluded.sql b/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_purge_bcomps_excluded.sql new file mode 100644 index 0000000000..b883a03f4f --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/support/subset_pg_purge_bcomps_excluded.sql @@ -0,0 +1,181 @@ +-- Compute BCOMPS-excluded corps ONCE in Postgres, then purge those corps from the corp-scoped COLIN extract tables. +-- +-- This is intended to replace the repeated Oracle-side "excluded_corps" computation previously embedded in every +-- transfer query (subset_transfer_chunk.sql). +-- +-- IMPORTANT: +-- - This script intentionally does NOT touch internal migration/processing tables (mig_*, corp_processing, +-- colin_tracking, affiliation_processing, etc). It only purges the corp-scoped COLIN extract tables +-- that are reloaded from Oracle. +-- - Because preserved processing/tracking tables still reference corporation/event rows, refresh mode must keep +-- FK enforcement suppressed across this purge window too (for example via replica_role, or by disabling +-- triggers on the preserved referencing tables too). +-- - This script avoids DO $$ blocks for DbSchemaCLI compatibility. +-- - The helper keyset tables are predeclared in the COLIN extract DDL and reused via TRUNCATE/INSERT. + +-- 1) Build keysets + +TRUNCATE TABLE colin_extract.subset_excluded_corp_parties; +TRUNCATE TABLE colin_extract.subset_excluded_events; +TRUNCATE TABLE colin_extract.subset_excluded_corps; + +INSERT INTO colin_extract.subset_excluded_corps (corp_num) +SELECT DISTINCT e.corp_num +FROM colin_extract.event e +JOIN colin_extract.filing f ON f.event_id = e.event_id +JOIN colin_extract.filing_user u ON u.event_id = e.event_id +WHERE e.corp_num IS NOT NULL + AND u.user_id = 'BCOMPS' + AND f.filing_type_cd IN ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC'); + +INSERT INTO colin_extract.subset_excluded_events (event_id) +SELECT DISTINCT e.event_id +FROM colin_extract.event e +JOIN colin_extract.subset_excluded_corps x ON x.corp_num = e.corp_num +WHERE e.event_id IS NOT NULL; + +INSERT INTO colin_extract.subset_excluded_corp_parties (corp_party_id) +SELECT DISTINCT cp.corp_party_id +FROM colin_extract.corp_party cp +JOIN colin_extract.subset_excluded_corps x ON x.corp_num = cp.corp_num +WHERE cp.corp_party_id IS NOT NULL; + +-- 2) Purge (delete child tables first) + +-- Event-scoped children +DELETE FROM colin_extract.notification_resend t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +DELETE FROM colin_extract.notification t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +DELETE FROM colin_extract.filing_user t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +DELETE FROM colin_extract.payment t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +DELETE FROM colin_extract.ledger_text t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +DELETE FROM colin_extract.conv_ledger t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +DELETE FROM colin_extract.conv_event t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +DELETE FROM colin_extract.completing_party t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +DELETE FROM colin_extract.submitting_party t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +DELETE FROM colin_extract.corp_involved_cont_in t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +DELETE FROM colin_extract.correction t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +DELETE FROM colin_extract.filing t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +-- corp_involved_amalgamating can reference corp_num via ted_corp_num/ting_corp_num as well as event_id. +-- Delete any rows where either side is excluded (covers non-event-owned references too). +DELETE FROM colin_extract.corp_involved_amalgamating t +USING colin_extract.subset_excluded_corps x +WHERE t.ted_corp_num = x.corp_num + OR t.ting_corp_num = x.corp_num; + +-- Corp-party related +DELETE FROM colin_extract.party_notification t +USING colin_extract.subset_excluded_corp_parties x +WHERE t.party_id = x.corp_party_id; + +DELETE FROM colin_extract.offices_held t +USING colin_extract.subset_excluded_corp_parties x +WHERE t.corp_party_id = x.corp_party_id; + +DELETE FROM colin_extract.corp_party_relationship t +USING colin_extract.subset_excluded_corp_parties x +WHERE t.corp_party_id = x.corp_party_id; + +DELETE FROM colin_extract.corp_party t +USING colin_extract.subset_excluded_corp_parties x +WHERE t.corp_party_id = x.corp_party_id; + +-- Corp-scoped tables +DELETE FROM colin_extract.office t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +DELETE FROM colin_extract.corp_name t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +DELETE FROM colin_extract.corp_state t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +DELETE FROM colin_extract.corp_comments t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +DELETE FROM colin_extract.corp_flag t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +DELETE FROM colin_extract.cont_out t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +DELETE FROM colin_extract.corp_restriction t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +DELETE FROM colin_extract.jurisdiction t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +DELETE FROM colin_extract.resolution t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +-- Share tables (delete deepest first) +DELETE FROM colin_extract.share_series t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +DELETE FROM colin_extract.share_struct_cls t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +DELETE FROM colin_extract.share_struct t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +-- Events last (many tables reference event_id) +DELETE FROM colin_extract.event t +USING colin_extract.subset_excluded_events x +WHERE t.event_id = x.event_id; + +-- Corporation last +DELETE FROM colin_extract.corporation t +USING colin_extract.subset_excluded_corps x +WHERE t.corp_num = x.corp_num; + +-- 3) Cleanup helper tables +TRUNCATE TABLE colin_extract.subset_excluded_corp_parties; +TRUNCATE TABLE colin_extract.subset_excluded_events; +TRUNCATE TABLE colin_extract.subset_excluded_corps; diff --git a/data-tool/scripts/_generated/subset_load_chunks/support/subset_transfer_cars.sql b/data-tool/scripts/_generated/subset_load_chunks/support/subset_transfer_cars.sql new file mode 100644 index 0000000000..93d8a87103 --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/support/subset_transfer_cars.sql @@ -0,0 +1,45 @@ +-- Global transfer of cars* tables from SOURCE Oracle DB (ctst) into TARGET Postgres extract DB (colin_extract schema). +-- Intended to be executed from a master DbSchemaCLI script connected to the target Postgres DB (colin_extract schema). +-- +-- These tables are NOT corp-scoped. The full dataset is transferred without filtering. +-- Volume is low enough that a full refresh is appropriate. + +transfer colin_extract.carsfile from ctst using +select + documtid, + filedate, + regiracf +from carsfile; + +transfer colin_extract.carsbox from ctst using +select + documtid, + accesnum, + batchnum, + boxrracf +from carsbox; + +transfer colin_extract.carsrept from ctst using +select + documtid, + docutype, + compnumb +from carsrept; + +transfer colin_extract.carindiv from ctst using +select + documtid, + replace(surname, CHR(0), '') as surname, + replace(firname, CHR(0), '') as firname, + replace(dircpoco, CHR(0), '') as dircpoco, + replace(dircflag, CHR(0), '') as dircflag, + replace(offiflag, CHR(0), '') as offiflag, + replace(chgreasn, CHR(0), '') as chgreasn, + replace(pfirname, CHR(0), '') as pfirname, + replace(psurname, CHR(0), '') as psurname, + replace(offtitle, CHR(0), '') as offtitle, + replace(dircaddr01, CHR(0), '') as dircaddr01, + replace(dircaddr02, CHR(0), '') as dircaddr02, + replace(dircaddr03, CHR(0), '') as dircaddr03, + replace(dircaddr04, CHR(0), '') as dircaddr04 +from carindiv; diff --git a/data-tool/scripts/_generated/subset_load_chunks/support/subset_transfer_chunk.sql b/data-tool/scripts/_generated/subset_load_chunks/support/subset_transfer_chunk.sql new file mode 100644 index 0000000000..2f27a78383 --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/support/subset_transfer_chunk.sql @@ -0,0 +1,1283 @@ +-- Transfer a chunk (or a whole subset) of corps from the SOURCE Oracle DB (ctst) into the TARGET Postgres extract DB (colin_extract schema). +-- +-- REQUIRED DbSchemaCLI variables (replace_variables=true): +-- target_corp_num_predicate : SQL predicate restricting the computed target_corp_num (NO trailing semicolon). +-- Examples: +-- target_corp_num in ('BC0460007','A1234567') +-- (target_corp_num in (...) OR target_corp_num in (...)) +-- oracle_corp_num_predicate : SQL predicate restricting Oracle corporation.corp_num (NO trailing semicolon). +-- Examples: +-- c.CORP_NUM in ('0460007','A1234567') +-- (c.CORP_NUM in (...) OR c.CORP_NUM in (...)) +-- oracle_corp_type_predicate : SQL predicate restricting Oracle corporation.corp_typ_cd (NO trailing semicolon). +-- Examples: +-- c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') +-- c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE','CP') +-- +-- Intended to be executed from a master DbSchemaCLI script connected to the target Postgres DB (colin_extract schema). +-- +-- IMPORTANT: +-- - This template intentionally avoids the boolean<->integer ALTER COLUMN hacks used in the full refresh script. +-- Instead, Oracle SELECTs emit boolean-friendly 't'/'f' strings for Postgres boolean columns. +-- - This template transfers corp-scoped tables only (no cars* tables). +-- +-- Performance notes: +-- - BCOMPS exclusion is NOT computed in Oracle in this template (to avoid repeating expensive Oracle-side joins per table). +-- Instead, load the requested corp set and purge BCOMPS-excluded corps ONCE in Postgres after the transfer suite completes +-- (see: subset_pg_purge_bcomps_excluded.sql). +-- - Joins are written to start from the subset (corporation_cte) to avoid "0 rows but slow" plans. +-- - ORDER BY clauses are removed (sorting is unnecessary overhead for transfers). +-- +-- Example (legacy vset mode): +-- vset target_corp_num_predicate=target_corp_num in ('BC1111585','BC1226175'); +-- vset oracle_corp_num_predicate=c.CORP_NUM in ('1111585','1226175'); + +-- corporation +transfer colin_extract.corporation from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +), +last_ar as ( + select e.corp_num, + to_number(to_char(max(date_1), 'YYYY')) as last_ar_reminder_year + from eml_log e + join rep_data r on r.param_id = e.param_id + and r.t20_1 = e.corp_num + join corp_list cl on cl.corp_num = e.corp_num + group by e.corp_num +) +select c.target_corp_num as CORP_NUM, + c.CORP_FROZEN_TYP_CD as corp_frozen_type_cd, + case + when c.CORP_TYP_CD in ('QA', 'QB', 'QC', 'QD', 'QE') then 'BC' + else c.CORP_TYP_CD + end as CORP_TYPE_CD, + c.CORP_PASSWORD, + c.RECOGNITION_DTS, + c.BN_9, + c.BN_15, + c.ADMIN_EMAIL, + c.ACCESSION_NUM, + c.LAST_AR_FILED_DT, + case c.SEND_AR_IND + when 'N' then 'f' + when 'Y' then 't' + else 't' + end as SEND_AR_IND, + la.last_ar_reminder_year as LAST_AR_REMINDER_YEAR +from corporation_cte c +left join last_ar la on la.corp_num = c.corp_num; + + +-- event +transfer colin_extract.event from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + c.target_corp_num as CORP_NUM, + e.event_typ_cd as event_type_cd, + e.event_timestmp as event_timerstamp, + e.trigger_dts +from corporation_cte c +join event e on e.corp_num = c.corp_num +-- not transferring BNUPD, ADDLEDGR events +where e.event_typ_cd not in ('BNUPD', 'ADDLEDGR'); + + +-- corp_name +transfer colin_extract.corp_name from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select c.target_corp_num as CORP_NUM, + cn.CORP_NAME_TYP_CD, + cn.start_event_id, + cn.end_event_id, + cn.CORP_NME as corp_name +from corporation_cte c +join CORP_NAME cn on cn.corp_num = c.corp_num; + + +-- corp_state +transfer colin_extract.corp_state from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select c.target_corp_num as CORP_NUM, + cs.STATE_TYP_CD as state_type_cd, + cos.op_state_typ_cd as op_state_type_cd, + cs.start_event_id, + cs.end_event_id +from corporation_cte c +join CORP_STATE cs on cs.corp_num = c.corp_num +join corp_op_state cos on cos.state_typ_cd = cs.state_typ_cd; + + +-- filing +transfer colin_extract.filing from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + f.filing_typ_cd as filing_type_cd, + f.effective_dt, + f.withdrawn_event_id, + trim(f.ods_typ_cd) as ods_type_cd, + f.nr_num, + f.COURT_ORDER_NUM, + f.CHANGE_DT, + f.PERIOD_END_DT, + case f.ARRANGEMENT_IND + when 'N' then 'f' + when 'Y' then 't' + else 't' + end as ARRANGEMENT_IND, + f.AUTH_SIGN_DT, + case f.COURT_APPR_IND + when 'N' then 'f' + when 'Y' then 't' + else 't' + end as COURT_APPR_IND +from corporation_cte c +join event e on e.corp_num = c.corp_num +join filing f on f.event_id = e.event_id; + + +-- filing_user +transfer colin_extract.filing_user from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + u.user_id, + u.last_nme as last_name, + u.first_nme as first_name, + u.middle_nme as middle_name, + u.email_addr, + u.BCOL_ACCT_NUM, + u.ROLE_TYP_CD +from corporation_cte c +join event e on e.corp_num = c.corp_num +join filing_user u on u.event_id = e.event_id; + + +-- address (shared/global table; stage then merge before loading dependents) +TRUNCATE TABLE colin_extract.subset_address_stage; + +transfer colin_extract.subset_address_stage from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select distinct + addr_id, + trim(province) as province, + trim(country_typ_cd) as country_typ_cd, + trim(replace(postal_cd, CHR(0), '')) as POSTAL_CD, + trim(addr_line_1) as addr_line_1, + trim(replace(addr_line_2, CHR(0), '')) as ADDR_LINE_2, + trim(addr_line_3) as addr_line_3, + trim(city) as city +from ( + select a.* + from corporation_cte c + join corp_party x on x.corp_num = c.corp_num + join address a on (x.delivery_addr_id = a.addr_id or x.mailing_addr_id = a.addr_id) + + UNION ALL + select a.* + from corporation_cte c + join office x on x.corp_num = c.corp_num + join address a on (x.delivery_addr_id = a.addr_id or x.mailing_addr_id = a.addr_id) + + UNION ALL + select a.* + from corporation_cte c + join event e on e.corp_num = c.corp_num + join completing_party x on x.event_id = e.event_id + join address a on x.mailing_addr_id = a.addr_id + + UNION ALL + select a.* + from corporation_cte c + join event e on e.corp_num = c.corp_num + join notification x on x.event_id = e.event_id + join address a on x.mailing_addr_id = a.addr_id + + UNION ALL + select a.* + from corporation_cte c + join event e on e.corp_num = c.corp_num + join notification_resend x on x.event_id = e.event_id + join address a on x.mailing_addr_id = a.addr_id + + UNION ALL + select a.* + from corporation_cte c + join event e on e.corp_num = c.corp_num + join submitting_party x on x.event_id = e.event_id + join address a on (x.notify_addr_id = a.addr_id or x.mailing_addr_id = a.addr_id) + + UNION ALL + select a.* + from corporation_cte c + join corp_party p on p.corp_num = c.corp_num + join party_notification x on x.party_id = p.corp_party_id + join address a on x.mailing_addr_id = a.addr_id +); + +INSERT INTO colin_extract.address ( + addr_id, + province, + country_typ_cd, + postal_cd, + addr_line_1, + addr_line_2, + addr_line_3, + city +) +SELECT s.addr_id, + s.province, + s.country_typ_cd, + s.postal_cd, + s.addr_line_1, + s.addr_line_2, + s.addr_line_3, + s.city +FROM ( + SELECT DISTINCT ON (addr_id) + addr_id, + province, + country_typ_cd, + postal_cd, + addr_line_1, + addr_line_2, + addr_line_3, + city + FROM colin_extract.subset_address_stage + WHERE addr_id IS NOT NULL + ORDER BY addr_id +) s +ON CONFLICT (addr_id) DO UPDATE +SET province = EXCLUDED.province, + country_typ_cd = EXCLUDED.country_typ_cd, + postal_cd = EXCLUDED.postal_cd, + addr_line_1 = EXCLUDED.addr_line_1, + addr_line_2 = EXCLUDED.addr_line_2, + addr_line_3 = EXCLUDED.addr_line_3, + city = EXCLUDED.city; + +TRUNCATE TABLE colin_extract.subset_address_stage; + + +-- office +transfer colin_extract.office from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select c.target_corp_num as CORP_NUM, + o.office_typ_cd, + o.start_event_id, + o.end_event_id, + o.mailing_addr_id, + o.delivery_addr_id +from corporation_cte c +join office o on o.corp_num = c.corp_num; + + +-- corp_comments +transfer colin_extract.corp_comments from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select cc.comment_dts, + c.target_corp_num as CORP_NUM, + cc.comments, + cc.USER_ID, + cc.FIRST_NME, + cc.LAST_NME, + cc.MIDDLE_NME, + cc.ACCESSION_COMMENTS +from corporation_cte c +join corp_comments cc on cc.corp_num = c.corp_num; + + +-- ledger_text +transfer colin_extract.ledger_text from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + l.notation, + l.USER_ID, + l.LEDGER_TEXT_DTS +from corporation_cte c +join event e on e.corp_num = c.corp_num +join ledger_text l on l.event_id = e.event_id; + + +-- corp_party +transfer colin_extract.corp_party from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select p.corp_party_id, + p.mailing_addr_id, + p.delivery_addr_id, + c.target_corp_num as CORP_NUM, + nvl(p.party_typ_cd, ' ') as party_typ_cd, + p.start_event_id, + p.end_event_id, + p.prev_party_id, + p.appointment_dt, + p.cessation_dt, + nvl(p.LAST_NME, ' ') as last_name, + nvl(p.MIDDLE_NME, ' ') as middle_name, + nvl(p.FIRST_NME, ' ') as first_name, + nvl(p.BUSINESS_NME, ' ') as business_name, + p.BUS_COMPANY_NUM, + p.CORR_TYP_CD, + p.OFFICE_NOTIFICATION_DT +from corporation_cte c +join corp_party p on p.corp_num = c.corp_num; + + +-- corp_party_relationship +transfer colin_extract.corp_party_relationship from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select cpr.CORP_PARTY_ID as corp_party_id, + cpr.RELATIONSHIP_TYP_CD as relationship_typ_cd +from corporation_cte c +join corp_party p on p.corp_num = c.corp_num +join CORP_PARTY_RELATIONSHIP cpr on cpr.corp_party_id = p.corp_party_id; + + +-- offices_held +transfer colin_extract.offices_held from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select oh.CORP_PARTY_ID as corp_party_id, + oh.OFFICER_TYP_CD as officer_typ_cd +from corporation_cte c +join corp_party p on p.corp_num = c.corp_num +join OFFICES_HELD oh on oh.corp_party_id = p.corp_party_id; + + +-- completing_party +transfer colin_extract.completing_party from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + cp.MAILING_ADDR_ID, + cp.FIRST_NME, + cp.LAST_NME, + cp.MIDDLE_NME, + cp.EMAIL_REQ_ADDRESS +from corporation_cte c +join event e on e.corp_num = c.corp_num +join completing_party cp on cp.event_id = e.event_id; + + +-- submitting_party +transfer colin_extract.submitting_party from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + sp.MAILING_ADDR_ID, + sp.NOTIFY_ADDR_ID, + sp.METHOD_TYP_CD, + sp.FIRST_NME, + sp.LAST_NME, + sp.MIDDLE_NME, + sp.EMAIL_REQ_ADDRESS, + sp.PICKUP_BY, + sp.BUSINESS_NME, + sp.NOTIFY_FIRST_NME, + sp.NOTIFY_LAST_NME, + sp.NOTIFY_MIDDLE_NME, + sp.PHONE_NUMBER +from corporation_cte c +join event e on e.corp_num = c.corp_num +join SUBMITTING_PARTY sp on sp.event_id = e.event_id; + + +-- corp_flag +transfer colin_extract.corp_flag from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select c.target_corp_num as CORP_NUM, + cf.CORP_FLAG_TYPE_CD, + cf.start_event_id, + cf.end_event_id +from corporation_cte c +join corp_flag cf on cf.corp_num = c.corp_num; + + +-- cont_out +transfer colin_extract.cont_out from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select c.target_corp_num as CORP_NUM, + co.CAN_JUR_TYP_CD, + co.CONT_OUT_DT, + co.OTHR_JURI_DESC, + co.HOME_COMPANY_NME, + co.start_event_id, + co.end_event_id +from corporation_cte c +join CONT_OUT co on co.corp_num = c.corp_num; + + +-- conv_event +transfer colin_extract.conv_event from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + ce.effective_dt, + case ce.REPORT_CORP_IND + when 'N' then 'f' + when 'Y' then 't' + else 't' + end as REPORT_CORP_IND, + ce.ACTIVITY_USER_ID, + ce.ACTIVITY_DT, + ce.ANNUAL_FILE_DT, + ce.ACCESSION_NUM, + ce.REMARKS +from corporation_cte c +join event e on e.corp_num = c.corp_num +join CONV_EVENT ce on ce.event_id = e.event_id; + + +-- conv_ledger +transfer colin_extract.conv_ledger from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + cl.LEDGER_TITLE_TXT, + cl.LEDGER_DESC, + cl.cars_docmnt_id +from corporation_cte c +join event e on e.corp_num = c.corp_num +join CONV_LEDGER cl on cl.event_id = e.event_id; + + +-- corp_involved - amalgamaTING_businesses +transfer colin_extract.corp_involved_amalgamating from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id as event_id, + c.target_corp_num as ted_corp_num, + case + when c2.corp_typ_cd in ('BC', 'ULC', 'CC') then 'BC' || c2.corp_num + else c2.corp_num + end as ting_corp_num, + ci.CORP_INVOLVE_ID as corp_involve_id, + ci.CAN_JUR_TYP_CD as can_jur_typ_cd, + case ci.ADOPTED_CORP_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as adopted_corp_ind, + ci.HOME_JURI_NUM as home_juri_num, + ci.OTHR_JURI_DESC as othr_juri_desc, + ci.FOREIGN_NME as foreign_nme +from corporation_cte c +join event e on e.corp_num = c.corp_num +join CORP_INVOLVED ci on ci.event_id = e.event_id +join corporation c2 on c2.corp_num = ci.corp_num +where e.event_typ_cd = 'CONVAMAL' +UNION ALL +select e.event_id as event_id, + c.target_corp_num as ted_corp_num, + case + when c2.corp_typ_cd in ('BC', 'ULC', 'CC') then 'BC' || c2.corp_num + else c2.corp_num + end as ting_corp_num, + ci.CORP_INVOLVE_ID as corp_involve_id, + ci.CAN_JUR_TYP_CD as can_jur_typ_cd, + case ci.ADOPTED_CORP_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as adopted_corp_ind, + ci.HOME_JURI_NUM as home_juri_num, + ci.OTHR_JURI_DESC as othr_juri_desc, + ci.FOREIGN_NME as foreign_nme +from corporation_cte c +join event e on e.corp_num = c.corp_num +join filing f on f.event_id = e.event_id +join CORP_INVOLVED ci on ci.event_id = e.event_id +join corporation c2 on c2.corp_num = ci.corp_num +where f.filing_typ_cd in ('AMALH', 'AMALV', 'AMALR', 'AMLHU', 'AMLVU', 'AMLRU', 'AMLHC', 'AMLVC', 'AMLRC'); + + +-- corp_involved - continue_in_historical_xpro +transfer colin_extract.corp_involved_cont_in from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + c.target_corp_num as CORP_NUM +from corporation_cte c +join event e on e.corp_num = c.corp_num +join filing f on f.event_id = e.event_id +where f.filing_typ_cd in ('CONTI', 'CONTU', 'CONTC') + and exists (select 1 from CORP_INVOLVED ci where ci.event_id = e.event_id); + + +-- corp_restriction +transfer colin_extract.corp_restriction from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select c.target_corp_num as CORP_NUM, + case cr.RESTRICTION_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as RESTRICTION_IND, + cr.start_event_id, + cr.end_event_id +from corporation_cte c +join CORP_RESTRICTION cr on cr.corp_num = c.corp_num; + + +-- correction +transfer colin_extract.correction from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + c.target_corp_num as CORP_NUM, + corr.ASSOCIATED_DOC_DESC +from corporation_cte c +join event e on e.corp_num = c.corp_num +join CORRECTION corr on corr.event_id = e.event_id; + + +-- continued_in_from_jurisdiction +transfer colin_extract.jurisdiction from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select c.target_corp_num as CORP_NUM, + j.CAN_JUR_TYP_CD, + j.XPRO_TYP_CD, + j.HOME_RECOGN_DT, + j.OTHR_JURIS_DESC, + j.HOME_JURIS_NUM, + j.BC_XPRO_NUM, + j.HOME_COMPANY_NME, + j.start_event_id +from corporation_cte c +join JURISDICTION j on j.corp_num = c.corp_num; + + +-- resolution +transfer colin_extract.resolution from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select c.target_corp_num as CORP_NUM, + r.RESOLUTION_DT, + r.RESOLUTION_TYPE_CODE, + r.start_event_id, + r.end_event_id +from corporation_cte c +join RESOLUTION r on r.corp_num = c.corp_num; + + +-- share_struct +transfer colin_extract.share_struct from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select c.target_corp_num as CORP_NUM, + ss.start_event_id, + ss.end_event_id +from corporation_cte c +join SHARE_STRUCT ss on ss.corp_num = c.corp_num; + + +-- share_struct_cls +transfer colin_extract.share_struct_cls from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select c.target_corp_num as CORP_NUM, + ssc.SHARE_CLASS_ID, + replace(ssc.CLASS_NME, CHR(0), '') as CLASS_NME, + ssc.CURRENCY_TYP_CD, + case ssc.MAX_SHARE_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as MAX_SHARE_IND, + ssc.SHARE_QUANTITY, + case ssc.SPEC_RIGHTS_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as SPEC_RIGHTS_IND, + case ssc.PAR_VALUE_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as PAR_VALUE_IND, + ssc.PAR_VALUE_AMT + 0 as PAR_VALUE_AMT, + ssc.OTHER_CURRENCY, + ssc.start_event_id +from corporation_cte c +join SHARE_STRUCT_CLS ssc on ssc.corp_num = c.corp_num; + + +-- share_series +transfer colin_extract.share_series from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select c.target_corp_num as CORP_NUM, + ss.SHARE_CLASS_ID, + ss.SERIES_ID, + case ss.MAX_SHARE_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as MAX_SHARE_IND, + ss.SHARE_QUANTITY, + case ss.SPEC_RIGHT_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as SPEC_RIGHT_IND, + ss.SERIES_NME, + ss.start_event_id +from corporation_cte c +join SHARE_SERIES ss on ss.corp_num = c.corp_num; + + +-- notification +transfer colin_extract.notification from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + n.METHOD_TYP_CD, + n.mailing_addr_id, + n.FIRST_NME, + n.LAST_NME, + n.MIDDLE_NME, + n.PICKUP_BY, + n.EMAIL_ADDRESS, + n.PHONE_NUMBER +from corporation_cte c +join event e on e.corp_num = c.corp_num +join NOTIFICATION n on n.event_id = e.event_id; + + +-- notification_resend +transfer colin_extract.notification_resend from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select e.event_id, + nr.METHOD_TYP_CD, + nr.mailing_addr_id, + nr.FIRST_NME, + nr.LAST_NME, + nr.MIDDLE_NME, + nr.PICKUP_BY, + nr.EMAIL_ADDRESS, + nr.PHONE_NUMBER +from corporation_cte c +join event e on e.corp_num = c.corp_num +join NOTIFICATION_RESEND nr on nr.event_id = e.event_id; + + +-- party_notification +transfer colin_extract.party_notification from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select pn.PARTY_ID, + pn.METHOD_TYP_CD, + pn.mailing_addr_id, + pn.FIRST_NME, + pn.LAST_NME, + pn.MIDDLE_NME, + pn.BUSINESS_NME, + pn.PICKUP_BY, + pn.EMAIL_ADDRESS, + pn.PHONE_NUMBER +from corporation_cte c +join corp_party cp on cp.corp_num = c.corp_num +join PARTY_NOTIFICATION pn on pn.party_id = cp.corp_party_id; + + +-- payment +transfer colin_extract.payment from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where &oracle_corp_num_predicate + and &oracle_corp_type_predicate + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where &target_corp_num_predicate +) +select p.event_id, + p.payment_typ_cd, + p.cc_holder_nme +from corporation_cte c +join event e on e.corp_num = c.corp_num +join payment p on p.event_id = e.event_id; diff --git a/data-tool/scripts/_generated/subset_load_chunks/transfer_all.sql b/data-tool/scripts/_generated/subset_load_chunks/transfer_all.sql new file mode 100644 index 0000000000..332964eba0 --- /dev/null +++ b/data-tool/scripts/_generated/subset_load_chunks/transfer_all.sql @@ -0,0 +1,1481 @@ +-- generated chunk script: transfer_all.sql +-- mode: load +-- chunk: 001/001 +-- target corps: 2 +-- oracle corp_num: 2 + +-- Transfer a chunk (or a whole subset) of corps from the SOURCE Oracle DB (ctst) into the TARGET Postgres extract DB (colin_extract schema). +-- +-- REQUIRED DbSchemaCLI variables (replace_variables=true): +-- target_corp_num_predicate : SQL predicate restricting the computed target_corp_num (NO trailing semicolon). +-- Examples: +-- target_corp_num in ('BC0460007','A1234567') +-- (target_corp_num in (...) OR target_corp_num in (...)) +-- oracle_corp_num_predicate : SQL predicate restricting Oracle corporation.corp_num (NO trailing semicolon). +-- Examples: +-- c.CORP_NUM in ('0460007','A1234567') +-- (c.CORP_NUM in (...) OR c.CORP_NUM in (...)) +-- oracle_corp_type_predicate : SQL predicate restricting Oracle corporation.corp_typ_cd (NO trailing semicolon). +-- Examples: +-- c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') +-- c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE','CP') +-- +-- Intended to be executed from a master DbSchemaCLI script connected to the target Postgres DB (colin_extract schema). +-- +-- IMPORTANT: +-- - This template intentionally avoids the boolean<->integer ALTER COLUMN hacks used in the full refresh script. +-- Instead, Oracle SELECTs emit boolean-friendly 't'/'f' strings for Postgres boolean columns. +-- - This template transfers corp-scoped tables only (no cars* tables). +-- +-- Performance notes: +-- - BCOMPS exclusion is NOT computed in Oracle in this template (to avoid repeating expensive Oracle-side joins per table). +-- Instead, load the requested corp set and purge BCOMPS-excluded corps ONCE in Postgres after the transfer suite completes +-- (see: subset_pg_purge_bcomps_excluded.sql). +-- - Joins are written to start from the subset (corporation_cte) to avoid "0 rows but slow" plans. +-- - ORDER BY clauses are removed (sorting is unnecessary overhead for transfers). +-- +-- Example (legacy vset mode): +-- vset target_corp_num_predicate=target_corp_num in ('BC1111585','BC1226175'); +-- vset oracle_corp_num_predicate=c.CORP_NUM in ('1111585','1226175'); + +-- corporation +transfer colin_extract.corporation from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +), +last_ar as ( + select e.corp_num, + to_number(to_char(max(date_1), 'YYYY')) as last_ar_reminder_year + from eml_log e + join rep_data r on r.param_id = e.param_id + and r.t20_1 = e.corp_num + join corp_list cl on cl.corp_num = e.corp_num + group by e.corp_num +) +select c.target_corp_num as CORP_NUM, + c.CORP_FROZEN_TYP_CD as corp_frozen_type_cd, + case + when c.CORP_TYP_CD in ('QA', 'QB', 'QC', 'QD', 'QE') then 'BC' + else c.CORP_TYP_CD + end as CORP_TYPE_CD, + c.CORP_PASSWORD, + c.RECOGNITION_DTS, + c.BN_9, + c.BN_15, + c.ADMIN_EMAIL, + c.ACCESSION_NUM, + c.LAST_AR_FILED_DT, + case c.SEND_AR_IND + when 'N' then 'f' + when 'Y' then 't' + else 't' + end as SEND_AR_IND, + la.last_ar_reminder_year as LAST_AR_REMINDER_YEAR +from corporation_cte c +left join last_ar la on la.corp_num = c.corp_num; + + +-- event +transfer colin_extract.event from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + c.target_corp_num as CORP_NUM, + e.event_typ_cd as event_type_cd, + e.event_timestmp as event_timerstamp, + e.trigger_dts +from corporation_cte c +join event e on e.corp_num = c.corp_num +-- not transferring BNUPD, ADDLEDGR events +where e.event_typ_cd not in ('BNUPD', 'ADDLEDGR'); + + +-- corp_name +transfer colin_extract.corp_name from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select c.target_corp_num as CORP_NUM, + cn.CORP_NAME_TYP_CD, + cn.start_event_id, + cn.end_event_id, + cn.CORP_NME as corp_name +from corporation_cte c +join CORP_NAME cn on cn.corp_num = c.corp_num; + + +-- corp_state +transfer colin_extract.corp_state from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select c.target_corp_num as CORP_NUM, + cs.STATE_TYP_CD as state_type_cd, + cos.op_state_typ_cd as op_state_type_cd, + cs.start_event_id, + cs.end_event_id +from corporation_cte c +join CORP_STATE cs on cs.corp_num = c.corp_num +join corp_op_state cos on cos.state_typ_cd = cs.state_typ_cd; + + +-- filing +transfer colin_extract.filing from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + f.filing_typ_cd as filing_type_cd, + f.effective_dt, + f.withdrawn_event_id, + trim(f.ods_typ_cd) as ods_type_cd, + f.nr_num, + f.COURT_ORDER_NUM, + f.CHANGE_DT, + f.PERIOD_END_DT, + case f.ARRANGEMENT_IND + when 'N' then 'f' + when 'Y' then 't' + else 't' + end as ARRANGEMENT_IND, + f.AUTH_SIGN_DT, + case f.COURT_APPR_IND + when 'N' then 'f' + when 'Y' then 't' + else 't' + end as COURT_APPR_IND +from corporation_cte c +join event e on e.corp_num = c.corp_num +join filing f on f.event_id = e.event_id; + + +-- filing_user +transfer colin_extract.filing_user from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + u.user_id, + u.last_nme as last_name, + u.first_nme as first_name, + u.middle_nme as middle_name, + u.email_addr, + u.BCOL_ACCT_NUM, + u.ROLE_TYP_CD +from corporation_cte c +join event e on e.corp_num = c.corp_num +join filing_user u on u.event_id = e.event_id; + + +-- address (shared/global table; stage then merge before loading dependents) +TRUNCATE TABLE colin_extract.subset_address_stage; + +transfer colin_extract.subset_address_stage from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select distinct + addr_id, + trim(province) as province, + trim(country_typ_cd) as country_typ_cd, + trim(replace(postal_cd, CHR(0), '')) as POSTAL_CD, + trim(addr_line_1) as addr_line_1, + trim(replace(addr_line_2, CHR(0), '')) as ADDR_LINE_2, + trim(addr_line_3) as addr_line_3, + trim(city) as city +from ( + select a.* + from corporation_cte c + join corp_party x on x.corp_num = c.corp_num + join address a on (x.delivery_addr_id = a.addr_id or x.mailing_addr_id = a.addr_id) + + UNION ALL + select a.* + from corporation_cte c + join office x on x.corp_num = c.corp_num + join address a on (x.delivery_addr_id = a.addr_id or x.mailing_addr_id = a.addr_id) + + UNION ALL + select a.* + from corporation_cte c + join event e on e.corp_num = c.corp_num + join completing_party x on x.event_id = e.event_id + join address a on x.mailing_addr_id = a.addr_id + + UNION ALL + select a.* + from corporation_cte c + join event e on e.corp_num = c.corp_num + join notification x on x.event_id = e.event_id + join address a on x.mailing_addr_id = a.addr_id + + UNION ALL + select a.* + from corporation_cte c + join event e on e.corp_num = c.corp_num + join notification_resend x on x.event_id = e.event_id + join address a on x.mailing_addr_id = a.addr_id + + UNION ALL + select a.* + from corporation_cte c + join event e on e.corp_num = c.corp_num + join submitting_party x on x.event_id = e.event_id + join address a on (x.notify_addr_id = a.addr_id or x.mailing_addr_id = a.addr_id) + + UNION ALL + select a.* + from corporation_cte c + join corp_party p on p.corp_num = c.corp_num + join party_notification x on x.party_id = p.corp_party_id + join address a on x.mailing_addr_id = a.addr_id +); + +INSERT INTO colin_extract.address ( + addr_id, + province, + country_typ_cd, + postal_cd, + addr_line_1, + addr_line_2, + addr_line_3, + city +) +SELECT s.addr_id, + s.province, + s.country_typ_cd, + s.postal_cd, + s.addr_line_1, + s.addr_line_2, + s.addr_line_3, + s.city +FROM ( + SELECT DISTINCT ON (addr_id) + addr_id, + province, + country_typ_cd, + postal_cd, + addr_line_1, + addr_line_2, + addr_line_3, + city + FROM colin_extract.subset_address_stage + WHERE addr_id IS NOT NULL + ORDER BY addr_id +) s +ON CONFLICT (addr_id) DO UPDATE +SET province = EXCLUDED.province, + country_typ_cd = EXCLUDED.country_typ_cd, + postal_cd = EXCLUDED.postal_cd, + addr_line_1 = EXCLUDED.addr_line_1, + addr_line_2 = EXCLUDED.addr_line_2, + addr_line_3 = EXCLUDED.addr_line_3, + city = EXCLUDED.city; + +TRUNCATE TABLE colin_extract.subset_address_stage; + + +-- office +transfer colin_extract.office from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select c.target_corp_num as CORP_NUM, + o.office_typ_cd, + o.start_event_id, + o.end_event_id, + o.mailing_addr_id, + o.delivery_addr_id +from corporation_cte c +join office o on o.corp_num = c.corp_num; + + +-- corp_comments +transfer colin_extract.corp_comments from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select cc.comment_dts, + c.target_corp_num as CORP_NUM, + cc.comments, + cc.USER_ID, + cc.FIRST_NME, + cc.LAST_NME, + cc.MIDDLE_NME, + cc.ACCESSION_COMMENTS +from corporation_cte c +join corp_comments cc on cc.corp_num = c.corp_num; + + +-- ledger_text +transfer colin_extract.ledger_text from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + l.notation, + l.USER_ID, + l.LEDGER_TEXT_DTS +from corporation_cte c +join event e on e.corp_num = c.corp_num +join ledger_text l on l.event_id = e.event_id; + + +-- corp_party +transfer colin_extract.corp_party from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select p.corp_party_id, + p.mailing_addr_id, + p.delivery_addr_id, + c.target_corp_num as CORP_NUM, + nvl(p.party_typ_cd, ' ') as party_typ_cd, + p.start_event_id, + p.end_event_id, + p.prev_party_id, + p.appointment_dt, + p.cessation_dt, + nvl(p.LAST_NME, ' ') as last_name, + nvl(p.MIDDLE_NME, ' ') as middle_name, + nvl(p.FIRST_NME, ' ') as first_name, + nvl(p.BUSINESS_NME, ' ') as business_name, + p.BUS_COMPANY_NUM, + p.CORR_TYP_CD, + p.OFFICE_NOTIFICATION_DT +from corporation_cte c +join corp_party p on p.corp_num = c.corp_num; + + +-- corp_party_relationship +transfer colin_extract.corp_party_relationship from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select cpr.CORP_PARTY_ID as corp_party_id, + cpr.RELATIONSHIP_TYP_CD as relationship_typ_cd +from corporation_cte c +join corp_party p on p.corp_num = c.corp_num +join CORP_PARTY_RELATIONSHIP cpr on cpr.corp_party_id = p.corp_party_id; + + +-- offices_held +transfer colin_extract.offices_held from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select oh.CORP_PARTY_ID as corp_party_id, + oh.OFFICER_TYP_CD as officer_typ_cd +from corporation_cte c +join corp_party p on p.corp_num = c.corp_num +join OFFICES_HELD oh on oh.corp_party_id = p.corp_party_id; + + +-- completing_party +transfer colin_extract.completing_party from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + cp.MAILING_ADDR_ID, + cp.FIRST_NME, + cp.LAST_NME, + cp.MIDDLE_NME, + cp.EMAIL_REQ_ADDRESS +from corporation_cte c +join event e on e.corp_num = c.corp_num +join completing_party cp on cp.event_id = e.event_id; + + +-- submitting_party +transfer colin_extract.submitting_party from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + sp.MAILING_ADDR_ID, + sp.NOTIFY_ADDR_ID, + sp.METHOD_TYP_CD, + sp.FIRST_NME, + sp.LAST_NME, + sp.MIDDLE_NME, + sp.EMAIL_REQ_ADDRESS, + sp.PICKUP_BY, + sp.BUSINESS_NME, + sp.NOTIFY_FIRST_NME, + sp.NOTIFY_LAST_NME, + sp.NOTIFY_MIDDLE_NME, + sp.PHONE_NUMBER +from corporation_cte c +join event e on e.corp_num = c.corp_num +join SUBMITTING_PARTY sp on sp.event_id = e.event_id; + + +-- corp_flag +transfer colin_extract.corp_flag from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select c.target_corp_num as CORP_NUM, + cf.CORP_FLAG_TYPE_CD, + cf.start_event_id, + cf.end_event_id +from corporation_cte c +join corp_flag cf on cf.corp_num = c.corp_num; + + +-- cont_out +transfer colin_extract.cont_out from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select c.target_corp_num as CORP_NUM, + co.CAN_JUR_TYP_CD, + co.CONT_OUT_DT, + co.OTHR_JURI_DESC, + co.HOME_COMPANY_NME, + co.start_event_id, + co.end_event_id +from corporation_cte c +join CONT_OUT co on co.corp_num = c.corp_num; + + +-- conv_event +transfer colin_extract.conv_event from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + ce.effective_dt, + case ce.REPORT_CORP_IND + when 'N' then 'f' + when 'Y' then 't' + else 't' + end as REPORT_CORP_IND, + ce.ACTIVITY_USER_ID, + ce.ACTIVITY_DT, + ce.ANNUAL_FILE_DT, + ce.ACCESSION_NUM, + ce.REMARKS +from corporation_cte c +join event e on e.corp_num = c.corp_num +join CONV_EVENT ce on ce.event_id = e.event_id; + + +-- conv_ledger +transfer colin_extract.conv_ledger from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + cl.LEDGER_TITLE_TXT, + cl.LEDGER_DESC, + cl.cars_docmnt_id +from corporation_cte c +join event e on e.corp_num = c.corp_num +join CONV_LEDGER cl on cl.event_id = e.event_id; + + +-- corp_involved - amalgamaTING_businesses +transfer colin_extract.corp_involved_amalgamating from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id as event_id, + c.target_corp_num as ted_corp_num, + case + when c2.corp_typ_cd in ('BC', 'ULC', 'CC') then 'BC' || c2.corp_num + else c2.corp_num + end as ting_corp_num, + ci.CORP_INVOLVE_ID as corp_involve_id, + ci.CAN_JUR_TYP_CD as can_jur_typ_cd, + case ci.ADOPTED_CORP_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as adopted_corp_ind, + ci.HOME_JURI_NUM as home_juri_num, + ci.OTHR_JURI_DESC as othr_juri_desc, + ci.FOREIGN_NME as foreign_nme +from corporation_cte c +join event e on e.corp_num = c.corp_num +join CORP_INVOLVED ci on ci.event_id = e.event_id +join corporation c2 on c2.corp_num = ci.corp_num +where e.event_typ_cd = 'CONVAMAL' +UNION ALL +select e.event_id as event_id, + c.target_corp_num as ted_corp_num, + case + when c2.corp_typ_cd in ('BC', 'ULC', 'CC') then 'BC' || c2.corp_num + else c2.corp_num + end as ting_corp_num, + ci.CORP_INVOLVE_ID as corp_involve_id, + ci.CAN_JUR_TYP_CD as can_jur_typ_cd, + case ci.ADOPTED_CORP_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as adopted_corp_ind, + ci.HOME_JURI_NUM as home_juri_num, + ci.OTHR_JURI_DESC as othr_juri_desc, + ci.FOREIGN_NME as foreign_nme +from corporation_cte c +join event e on e.corp_num = c.corp_num +join filing f on f.event_id = e.event_id +join CORP_INVOLVED ci on ci.event_id = e.event_id +join corporation c2 on c2.corp_num = ci.corp_num +where f.filing_typ_cd in ('AMALH', 'AMALV', 'AMALR', 'AMLHU', 'AMLVU', 'AMLRU', 'AMLHC', 'AMLVC', 'AMLRC'); + + +-- corp_involved - continue_in_historical_xpro +transfer colin_extract.corp_involved_cont_in from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + c.target_corp_num as CORP_NUM +from corporation_cte c +join event e on e.corp_num = c.corp_num +join filing f on f.event_id = e.event_id +where f.filing_typ_cd in ('CONTI', 'CONTU', 'CONTC') + and exists (select 1 from CORP_INVOLVED ci where ci.event_id = e.event_id); + + +-- corp_restriction +transfer colin_extract.corp_restriction from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select c.target_corp_num as CORP_NUM, + case cr.RESTRICTION_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as RESTRICTION_IND, + cr.start_event_id, + cr.end_event_id +from corporation_cte c +join CORP_RESTRICTION cr on cr.corp_num = c.corp_num; + + +-- correction +transfer colin_extract.correction from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + c.target_corp_num as CORP_NUM, + corr.ASSOCIATED_DOC_DESC +from corporation_cte c +join event e on e.corp_num = c.corp_num +join CORRECTION corr on corr.event_id = e.event_id; + + +-- continued_in_from_jurisdiction +transfer colin_extract.jurisdiction from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select c.target_corp_num as CORP_NUM, + j.CAN_JUR_TYP_CD, + j.XPRO_TYP_CD, + j.HOME_RECOGN_DT, + j.OTHR_JURIS_DESC, + j.HOME_JURIS_NUM, + j.BC_XPRO_NUM, + j.HOME_COMPANY_NME, + j.start_event_id +from corporation_cte c +join JURISDICTION j on j.corp_num = c.corp_num; + + +-- resolution +transfer colin_extract.resolution from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select c.target_corp_num as CORP_NUM, + r.RESOLUTION_DT, + r.RESOLUTION_TYPE_CODE, + r.start_event_id, + r.end_event_id +from corporation_cte c +join RESOLUTION r on r.corp_num = c.corp_num; + + +-- share_struct +transfer colin_extract.share_struct from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select c.target_corp_num as CORP_NUM, + ss.start_event_id, + ss.end_event_id +from corporation_cte c +join SHARE_STRUCT ss on ss.corp_num = c.corp_num; + + +-- share_struct_cls +transfer colin_extract.share_struct_cls from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select c.target_corp_num as CORP_NUM, + ssc.SHARE_CLASS_ID, + replace(ssc.CLASS_NME, CHR(0), '') as CLASS_NME, + ssc.CURRENCY_TYP_CD, + case ssc.MAX_SHARE_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as MAX_SHARE_IND, + ssc.SHARE_QUANTITY, + case ssc.SPEC_RIGHTS_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as SPEC_RIGHTS_IND, + case ssc.PAR_VALUE_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as PAR_VALUE_IND, + ssc.PAR_VALUE_AMT + 0 as PAR_VALUE_AMT, + ssc.OTHER_CURRENCY, + ssc.start_event_id +from corporation_cte c +join SHARE_STRUCT_CLS ssc on ssc.corp_num = c.corp_num; + + +-- share_series +transfer colin_extract.share_series from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select c.target_corp_num as CORP_NUM, + ss.SHARE_CLASS_ID, + ss.SERIES_ID, + case ss.MAX_SHARE_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as MAX_SHARE_IND, + ss.SHARE_QUANTITY, + case ss.SPEC_RIGHT_IND + when 'N' then 'f' + when 'Y' then 't' + else 'f' + end as SPEC_RIGHT_IND, + ss.SERIES_NME, + ss.start_event_id +from corporation_cte c +join SHARE_SERIES ss on ss.corp_num = c.corp_num; + + +-- notification +transfer colin_extract.notification from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + n.METHOD_TYP_CD, + n.mailing_addr_id, + n.FIRST_NME, + n.LAST_NME, + n.MIDDLE_NME, + n.PICKUP_BY, + n.EMAIL_ADDRESS, + n.PHONE_NUMBER +from corporation_cte c +join event e on e.corp_num = c.corp_num +join NOTIFICATION n on n.event_id = e.event_id; + + +-- notification_resend +transfer colin_extract.notification_resend from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select e.event_id, + nr.METHOD_TYP_CD, + nr.mailing_addr_id, + nr.FIRST_NME, + nr.LAST_NME, + nr.MIDDLE_NME, + nr.PICKUP_BY, + nr.EMAIL_ADDRESS, + nr.PHONE_NUMBER +from corporation_cte c +join event e on e.corp_num = c.corp_num +join NOTIFICATION_RESEND nr on nr.event_id = e.event_id; + + +-- party_notification +transfer colin_extract.party_notification from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select pn.PARTY_ID, + pn.METHOD_TYP_CD, + pn.mailing_addr_id, + pn.FIRST_NME, + pn.LAST_NME, + pn.MIDDLE_NME, + pn.BUSINESS_NME, + pn.PICKUP_BY, + pn.EMAIL_ADDRESS, + pn.PHONE_NUMBER +from corporation_cte c +join corp_party cp on cp.corp_num = c.corp_num +join PARTY_NOTIFICATION pn on pn.party_id = cp.corp_party_id; + + +-- payment +transfer colin_extract.payment from ctst using +with corp_list as ( + select /*+ materialize */ c.corp_num + from corporation c + where c.CORP_NUM in ( + '0008367', + '0008368' +) + and c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +), +corporation_cte as ( + select * + from ( + select c.*, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end as target_corp_num + from corporation c + join corp_list cl on cl.corp_num = c.corp_num + ) + where target_corp_num in ( + 'BC0008367', + 'BC0008368' +) +) +select p.event_id, + p.payment_typ_cd, + p.cc_holder_nme +from corporation_cte c +join event e on e.corp_num = c.corp_num +join payment p on p.event_id = e.event_id; diff --git a/data-tool/scripts/colin_corps_extract_postgres_views_ddl b/data-tool/scripts/colin_corps_extract_postgres_views_ddl index 55fe7734c2..9a3b23ae41 100644 --- a/data-tool/scripts/colin_corps_extract_postgres_views_ddl +++ b/data-tool/scripts/colin_corps_extract_postgres_views_ddl @@ -1,6 +1,14 @@ -- The following views are used mainly to help with bad data analysis and to help with determining eligibility of -- groups & batches of businesses we can migrate at specific points in time. +\if :{?schema_name} +\else +\set schema_name public +\endif + +CREATE SCHEMA IF NOT EXISTS :"schema_name"; +SET search_path TO :"schema_name", public; + CREATE MATERIALIZED VIEW mv_corps_with_officers AS select distinct cp.corp_num from corporation c diff --git a/data-tool/scripts/generate_cprd_subset_extract.py b/data-tool/scripts/generate_cprd_subset_extract.py index 479aba2f58..ff949861a7 100644 --- a/data-tool/scripts/generate_cprd_subset_extract.py +++ b/data-tool/scripts/generate_cprd_subset_extract.py @@ -31,7 +31,7 @@ import argparse import re -from dataclasses import dataclass +from dataclasses import dataclass, replace from enum import Enum from pathlib import Path from typing import Dict, Iterable, List, Sequence @@ -86,6 +86,7 @@ class cfg_GenerationConfig: out_master: Path out_chunks_dir: Path + source_connection: str target_connection: str target_schema: str @@ -98,6 +99,10 @@ class cfg_GenerationConfig: TMPL_TOKEN_TARGET_PRED = "&target_corp_num_predicate" # used by transfer template (Oracle-side) TMPL_TOKEN_ORACLE_PRED = "&oracle_corp_num_predicate" # used by transfer template (Oracle-side) TMPL_TOKEN_ORACLE_CORP_TYPE_PRED = "&oracle_corp_type_predicate" # used by transfer template (Oracle-side) +TMPL_TOKEN_SOURCE_CONNECTION = "__DBSCHEMA_SOURCE_CONNECTION__" # rendered generator-time source DbSchema alias +TMPL_TOKEN_TARGET_SCHEMA = "__DBSCHEMA_TARGET_SCHEMA__" # rendered generator-time target Postgres schema +TMPL_CONNECTION_TOKENS = (TMPL_TOKEN_SOURCE_CONNECTION, TMPL_TOKEN_TARGET_SCHEMA) +DBSCHEMA_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") @dataclass(frozen=True) @@ -306,30 +311,37 @@ def tmpl_default_bundle(repo_root: Path) -> tmpl_TemplateBundle: pg_prepare_address_stage = tmpl_TemplateSpec( name="subset_pg_prepare_address_stage", path=subset_dir / "subset_pg_prepare_address_stage.sql", + required_tokens=(TMPL_TOKEN_TARGET_SCHEMA,), ) pg_cleanup_address_stage = tmpl_TemplateSpec( name="subset_pg_cleanup_address_stage", path=subset_dir / "subset_pg_cleanup_address_stage.sql", + required_tokens=(TMPL_TOKEN_TARGET_SCHEMA,), ) pg_cleanup_orphan_children = tmpl_TemplateSpec( name="subset_pg_cleanup_orphan_children", path=subset_dir / "subset_pg_cleanup_orphan_children.sql", + required_tokens=(TMPL_TOKEN_TARGET_SCHEMA,), ) disable_triggers = tmpl_TemplateSpec( name="subset_disable_triggers", path=subset_dir / "subset_disable_triggers.sql", + required_tokens=(TMPL_TOKEN_TARGET_SCHEMA,), ) enable_triggers = tmpl_TemplateSpec( name="subset_enable_triggers", path=subset_dir / "subset_enable_triggers.sql", + required_tokens=(TMPL_TOKEN_TARGET_SCHEMA,), ) pg_boolean_casts = tmpl_TemplateSpec( name="subset_pg_boolean_casts", path=subset_dir / "subset_pg_boolean_casts.sql", + required_tokens=(TMPL_TOKEN_TARGET_SCHEMA,), ) pg_fastload_begin = tmpl_TemplateSpec( name="subset_pg_fastload_begin", path=subset_dir / "subset_pg_fastload_begin.sql", + required_tokens=(TMPL_TOKEN_TARGET_SCHEMA,), ) pg_fastload_end = tmpl_TemplateSpec( name="subset_pg_fastload_end", @@ -338,24 +350,33 @@ def tmpl_default_bundle(repo_root: Path) -> tmpl_TemplateBundle: pg_purge_bcomps_excluded = tmpl_TemplateSpec( name="subset_pg_purge_bcomps_excluded", path=subset_dir / "subset_pg_purge_bcomps_excluded.sql", + required_tokens=(TMPL_TOKEN_TARGET_SCHEMA,), ) delete_chunk = tmpl_TemplateSpec( name="subset_delete_chunk", path=subset_dir / "subset_delete_chunk.sql", - required_tokens=(TMPL_TOKEN_CORP_IDS,), + required_tokens=(TMPL_TOKEN_CORP_IDS, TMPL_TOKEN_TARGET_SCHEMA), ) transfer_chunk = tmpl_TemplateSpec( name="subset_transfer_chunk", path=subset_dir / "subset_transfer_chunk.sql", - required_tokens=(TMPL_TOKEN_TARGET_PRED, TMPL_TOKEN_ORACLE_PRED, TMPL_TOKEN_ORACLE_CORP_TYPE_PRED), + required_tokens=( + TMPL_TOKEN_TARGET_PRED, + TMPL_TOKEN_ORACLE_PRED, + TMPL_TOKEN_ORACLE_CORP_TYPE_PRED, + TMPL_TOKEN_SOURCE_CONNECTION, + TMPL_TOKEN_TARGET_SCHEMA, + ), ) delete_cars = tmpl_TemplateSpec( name="subset_delete_cars", path=subset_dir / "subset_delete_cars.sql", + required_tokens=(TMPL_TOKEN_TARGET_SCHEMA,), ) transfer_cars = tmpl_TemplateSpec( name="subset_transfer_cars", path=subset_dir / "subset_transfer_cars.sql", + required_tokens=(TMPL_TOKEN_SOURCE_CONNECTION, TMPL_TOKEN_TARGET_SCHEMA), ) return tmpl_TemplateBundle( @@ -404,6 +425,44 @@ def tmpl_render(template_text: str, *, replacements: Dict[str, str]) -> str: out = out.replace(token, value) return out +def cfg_validate_dbschema_identifier(name: str, value: str) -> str: + if not value: + raise SystemExit(f"{name} must not be empty") + if not DBSCHEMA_IDENTIFIER_RE.match(value): + raise SystemExit( + f"{name} must be a conservative DbSchema/Postgres identifier using letters, digits, " + "and underscore, and must not start with a digit" + ) + return value + + +def cfg_validate_pg_schema_identifier(name: str, value: str) -> str: + value = cfg_validate_dbschema_identifier(name, value) + if value != value.lower(): + raise SystemExit( + f"{name} must be lowercase. PostgreSQL folds unquoted uppercase identifiers to lowercase, " + "so uppercase schema values are rejected to avoid mismatches." + ) + return value + + +def tmpl_connection_replacements(cfg: cfg_GenerationConfig) -> Dict[str, str]: + return { + TMPL_TOKEN_SOURCE_CONNECTION: cfg.source_connection, + TMPL_TOKEN_TARGET_SCHEMA: cfg.target_schema, + } + + +def tmpl_assert_no_connection_tokens(spec: tmpl_TemplateSpec, rendered_text: str) -> None: + remaining = [token for token in TMPL_CONNECTION_TOKENS if token in rendered_text] + if remaining: + raise SystemExit( + "Template source/schema token rendering failed.\n" + f"Template: {spec.name}\n" + f"Path: {spec.path}\n" + f"Remaining token(s): {', '.join(remaining)}\n" + ) + # ========================= # chunk_* (chunk planning) @@ -446,9 +505,94 @@ def chunk_plan_chunks( # ========================= def gen_write_text(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding="utf-8") +def gen_write_rendered_connection_template( + *, + cfg: cfg_GenerationConfig, + spec: tmpl_TemplateSpec, + output_dir: Path, +) -> tmpl_TemplateSpec: + template_text = tmpl_load_text(spec) + rendered_text = tmpl_render(template_text, replacements=tmpl_connection_replacements(cfg)) + tmpl_assert_no_connection_tokens(spec, rendered_text) + rendered_path = output_dir / spec.path.name + gen_write_text(rendered_path, rendered_text) + return replace( + spec, + path=rendered_path, + required_tokens=tuple(token for token in spec.required_tokens if token not in TMPL_CONNECTION_TOKENS), + ) + + +def gen_write_rendered_connection_templates( + *, + cfg: cfg_GenerationConfig, + templates: tmpl_TemplateBundle, +) -> tmpl_TemplateBundle: + support_dir = cfg.out_chunks_dir / "support" + return replace( + templates, + pg_prepare_address_stage=gen_write_rendered_connection_template( + cfg=cfg, + spec=templates.pg_prepare_address_stage, + output_dir=support_dir, + ), + pg_cleanup_address_stage=gen_write_rendered_connection_template( + cfg=cfg, + spec=templates.pg_cleanup_address_stage, + output_dir=support_dir, + ), + pg_cleanup_orphan_children=gen_write_rendered_connection_template( + cfg=cfg, + spec=templates.pg_cleanup_orphan_children, + output_dir=support_dir, + ), + disable_triggers=gen_write_rendered_connection_template( + cfg=cfg, + spec=templates.disable_triggers, + output_dir=support_dir, + ), + enable_triggers=gen_write_rendered_connection_template( + cfg=cfg, + spec=templates.enable_triggers, + output_dir=support_dir, + ), + pg_boolean_casts=gen_write_rendered_connection_template( + cfg=cfg, + spec=templates.pg_boolean_casts, + output_dir=support_dir, + ), + pg_purge_bcomps_excluded=gen_write_rendered_connection_template( + cfg=cfg, + spec=templates.pg_purge_bcomps_excluded, + output_dir=support_dir, + ), + delete_chunk=gen_write_rendered_connection_template( + cfg=cfg, + spec=templates.delete_chunk, + output_dir=support_dir, + ), + transfer_chunk=gen_write_rendered_connection_template( + cfg=cfg, + spec=templates.transfer_chunk, + output_dir=support_dir, + ), + delete_cars=gen_write_rendered_connection_template( + cfg=cfg, + spec=templates.delete_cars, + output_dir=support_dir, + ), + transfer_cars=gen_write_rendered_connection_template( + cfg=cfg, + spec=templates.transfer_cars, + output_dir=support_dir, + ), + ) + + def gen_build_chunk_sql( *, chunk: chunk_ChunkSpec, @@ -497,7 +641,9 @@ def gen_build_chunk_sql( rendered_transfer = tmpl_render(transfer_template_text, replacements=replacements) if (TMPL_TOKEN_TARGET_PRED in rendered_transfer or TMPL_TOKEN_ORACLE_PRED in rendered_transfer or - TMPL_TOKEN_ORACLE_CORP_TYPE_PRED in rendered_transfer): + TMPL_TOKEN_ORACLE_CORP_TYPE_PRED in rendered_transfer or + TMPL_TOKEN_SOURCE_CONNECTION in rendered_transfer or + TMPL_TOKEN_TARGET_SCHEMA in rendered_transfer): raise SystemExit( f"Internal error: token(s) remained after rendering transfer template for chunk {chunk.index:03d}." ) @@ -561,9 +707,12 @@ def _gen_emit_pg_disable_begin(lines: List[str], *, cfg: cfg_GenerationConfig, t lines.append(f"execute {templates.disable_triggers.path.as_posix()}") if cfg.mode == cfg_GenerationMode.REFRESH: lines.append("-- Refresh-only: preserved processing/tracking tables still reference corporation/event rows.") - lines.append("ALTER TABLE corp_processing DISABLE TRIGGER ALL;") - # lines.append("ALTER TABLE auth_processing DISABLE TRIGGER ALL;") - lines.append("ALTER TABLE colin_tracking DISABLE TRIGGER ALL;") + lines.append(f"ALTER TABLE {cfg.target_schema}.corp_processing DISABLE TRIGGER ALL;") + lines.append("-- Deferred: DDL/docs identify auth_processing and affiliation_processing as preserved FK-owning") + lines.append("-- tables too, but auth_processing was introduced here as commented-out with no repo-history") + lines.append("-- rationale. Do not extend table-trigger suppression until Cloud SQL privilege/runtime") + lines.append("-- validation confirms the intended preserved-table set.") + lines.append(f"ALTER TABLE {cfg.target_schema}.colin_tracking DISABLE TRIGGER ALL;") lines.append("") return @@ -581,9 +730,9 @@ def _gen_emit_pg_disable_end(lines: List[str], *, cfg: cfg_GenerationConfig, tem lines.append(f"execute {templates.enable_triggers.path.as_posix()}") if cfg.mode == cfg_GenerationMode.REFRESH: lines.append("-- Refresh-only: restore preserved processing/tracking table triggers too.") - lines.append("ALTER TABLE corp_processing ENABLE TRIGGER ALL;") - # lines.append("ALTER TABLE auth_processing ENABLE TRIGGER ALL;") - lines.append("ALTER TABLE colin_tracking ENABLE TRIGGER ALL;") + lines.append(f"ALTER TABLE {cfg.target_schema}.corp_processing ENABLE TRIGGER ALL;") + lines.append("-- Deferred: see matching disable-side note for auth_processing/affiliation_processing.") + lines.append(f"ALTER TABLE {cfg.target_schema}.colin_tracking ENABLE TRIGGER ALL;") lines.append("") return @@ -646,8 +795,8 @@ def gen_build_master_script_inline( lines.append(f"learn schema {cfg.target_schema};") lines.append("") - lines.append("truncate table public.colin_extract_version; " - "insert into public.colin_extract_version (extracted_at) values (current_timestamp); " + lines.append(f"truncate table {cfg.target_schema}.colin_extract_version; " + f"insert into {cfg.target_schema}.colin_extract_version (extracted_at) values (current_timestamp); " ) lines.append("") @@ -750,8 +899,8 @@ def gen_build_master_script_vset( lines.append(f"learn schema {cfg.target_schema};") lines.append("") - lines.append("truncate table public.colin_extract_version; " - "insert into public.colin_extract_version (extracted_at) values (current_timestamp); " + lines.append(f"truncate table {cfg.target_schema}.colin_extract_version; " + f"insert into {cfg.target_schema}.colin_extract_version (extracted_at) values (current_timestamp); " ) lines.append("") @@ -773,10 +922,6 @@ def gen_build_master_script_vset( _gen_emit_pg_disable_begin(lines, cfg=cfg, templates=templates) _gen_emit_refresh_fk_note(lines, cfg=cfg) - if cfg.mode == cfg_GenerationMode.REFRESH: - lines.append("-- Cleanup stale orphan child rows before chunked refresh deletes.") - lines.append(f"execute {templates.pg_cleanup_orphan_children.path.as_posix()}") - lines.append("") if cfg.include_cars: lines.append("-- global cars* refresh (not corp-scoped; full dataset truncate + reload)") @@ -909,6 +1054,12 @@ def cli_parse_args(argv: List[str] | None = None) -> argparse.Namespace: action="store_true", help="If set, any all-numeric corp id lines will be normalized to BC for the TARGET/Postgres corp_num.", ) + parser.add_argument( + "--include-cars", + dest="include_cars", + action="store_true", + help="Include global cars* refresh step (carsfile/carsbox/carsrept/carindiv; generator default).", + ) parser.add_argument( "--no-cars", dest="include_cars", @@ -951,6 +1102,12 @@ def cli_parse_args(argv: List[str] | None = None) -> argparse.Namespace: "session_replication_role in the master script and nested execute files.", ) + parser.add_argument( + "--source-connection", + default="cprd", + help="DbSchemaCLI connection name for the SOURCE Oracle DB (default: cprd).", + ) + parser.add_argument( "--target-connection", default="cprd_pg_subset", @@ -994,16 +1151,19 @@ def cfg_build_config(args: argparse.Namespace) -> cfg_GenerationConfig: ) out_master.parent.mkdir(parents=True, exist_ok=True) - # Chunk scripts dir is always derived from master output stem for determinism. + # Chunk/support scripts dir is always derived from master output stem for determinism. out_chunks_dir = out_master.parent / f"{out_master.stem}_chunks" - if render_mode == cfg_RenderMode.INLINE: - out_chunks_dir.mkdir(parents=True, exist_ok=True) + out_chunks_dir.mkdir(parents=True, exist_ok=True) if args.chunk_size <= 0: raise SystemExit("--chunk-size must be > 0") if args.threads <= 0: raise SystemExit("--threads must be > 0") + source_connection = cfg_validate_dbschema_identifier("--source-connection", str(args.source_connection)) + target_connection = cfg_validate_dbschema_identifier("--target-connection", str(args.target_connection)) + target_schema = cfg_validate_pg_schema_identifier("--target-schema", str(args.target_schema)) + return cfg_GenerationConfig( repo_root=repo_root, corp_file=corp_file, @@ -1021,8 +1181,9 @@ def cfg_build_config(args: argparse.Namespace) -> cfg_GenerationConfig: or_of_in_max_ids=or_of_in_max_ids, out_master=out_master, out_chunks_dir=out_chunks_dir, - target_connection=str(args.target_connection), - target_schema=str(args.target_schema), + source_connection=source_connection, + target_connection=target_connection, + target_schema=target_schema, ) @@ -1034,6 +1195,7 @@ def _effective_oracle_strategy(cfg: cfg_GenerationConfig, total_ids: int) -> cfg def run(cfg: cfg_GenerationConfig) -> int: templates = tmpl_default_bundle(cfg.repo_root) + templates = gen_write_rendered_connection_templates(cfg=cfg, templates=templates) if cfg.pg_debug_session_probes and cfg.render_mode != cfg_RenderMode.INLINE: raise SystemExit("--pg-debug-session-probes currently supports only --render-mode inline.") @@ -1157,6 +1319,9 @@ def run(cfg: cfg_GenerationConfig) -> int: print(" - Corp ids in the file should match the TARGET Postgres extract corp_num format (e.g. BC0460007).") print(" - If you have numeric-only corp ids, consider --prefix-numeric-bc.") print(f" - corp ids: {n_ids} => ceil({n_ids}/{cfg.chunk_size}) = {in_groups} chunk(s)") + print(f" - Source DbSchema connection: {cfg.source_connection}") + print(f" - Target DbSchema connection: {cfg.target_connection}") + print(f" - Target schema: {cfg.target_schema}") print(f" - Oracle IN-list handling: {effective_strategy.value} (configured: {cfg.oracle_in_strategy.value})") print(f" - chunk-size (max items per IN list): {cfg.chunk_size}") if effective_strategy == cfg_OracleInStrategy.CHUNK_FILES: @@ -1176,7 +1341,7 @@ def run(cfg: cfg_GenerationConfig) -> int: print(f" - Postgres fast-load session settings: {'ENABLED' if cfg.pg_fastload else 'disabled'} (--pg-fastload)") print(f" - Postgres trigger suppression: {cfg.pg_disable_method.value} (--pg-disable-method)") print(" - subset runs acquire a session-level advisory lock on the target DB to prevent overlap.") - print(" - Address loads use the predeclared helper table public.subset_address_stage and merge into public.address by addr_id.") + print(f" - Address loads use the predeclared helper table {cfg.target_schema}.subset_address_stage and merge into {cfg.target_schema}.address by addr_id.") print(" - BCOMPS purge keysets also use predeclared helper tables in the extract schema (subset_excluded_*).") print(" - subset runs should not overlap on the same target DB, and the runtime role must be able to truncate/read/write those helper tables.") if cfg.pg_debug_session_probes: @@ -1215,6 +1380,9 @@ def run(cfg: cfg_GenerationConfig) -> int: print("Notes:") print(" - This script relies on DbSchemaCLI vset variables and runtime substitution.") print(f" - corp ids: {n_ids} => ceil({n_ids}/{cfg.chunk_size}) = {in_groups} chunk(s)") + print(f" - Source DbSchema connection: {cfg.source_connection}") + print(f" - Target DbSchema connection: {cfg.target_connection}") + print(f" - Target schema: {cfg.target_schema}") print(f" - Oracle IN-list handling: {effective_strategy.value} (configured: {cfg.oracle_in_strategy.value})") print(f" - chunk-size (max items per IN list): {cfg.chunk_size}") if effective_strategy == cfg_OracleInStrategy.CHUNK_FILES: @@ -1230,7 +1398,7 @@ def run(cfg: cfg_GenerationConfig) -> int: print(f" - Postgres fast-load session settings: {'ENABLED' if cfg.pg_fastload else 'disabled'} (--pg-fastload)") print(f" - Postgres trigger suppression: {cfg.pg_disable_method.value} (--pg-disable-method)") print(" - subset runs acquire a session-level advisory lock on the target DB to prevent overlap.") - print(" - Address loads use the predeclared helper table public.subset_address_stage and merge into public.address by addr_id.") + print(f" - Address loads use the predeclared helper table {cfg.target_schema}.subset_address_stage and merge into {cfg.target_schema}.address by addr_id.") print(" - BCOMPS purge keysets also use predeclared helper tables in the extract schema (subset_excluded_*).") print(" - subset runs should not overlap on the same target DB, and the runtime role must be able to truncate/read/write those helper tables.") if cfg.pg_debug_session_probes: diff --git a/data-tool/scripts/subset/subset_delete_cars.sql b/data-tool/scripts/subset/subset_delete_cars.sql index ebcbec0ac5..c7b41a6c87 100644 --- a/data-tool/scripts/subset/subset_delete_cars.sql +++ b/data-tool/scripts/subset/subset_delete_cars.sql @@ -4,7 +4,7 @@ -- These tables are NOT corp-scoped, so we truncate the entire dataset and reload from Oracle. -- Volume is low enough that a full refresh is appropriate. -TRUNCATE TABLE carindiv; -TRUNCATE TABLE carsrept; -TRUNCATE TABLE carsbox; -TRUNCATE TABLE carsfile; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.carindiv; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.carsrept; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.carsbox; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.carsfile; diff --git a/data-tool/scripts/subset/subset_delete_chunk.sql b/data-tool/scripts/subset/subset_delete_chunk.sql index 45ed54a76d..89dbb71ddf 100644 --- a/data-tool/scripts/subset/subset_delete_chunk.sql +++ b/data-tool/scripts/subset/subset_delete_chunk.sql @@ -4,7 +4,7 @@ -- corp_ids_in : comma-separated SQL string literals for target corp_num values (NO parentheses), -- e.g. 'BC0460007','A1234567' -- --- Intended to be executed from a master DbSchemaCLI script connected to the target Postgres DB (cprd_pg). +-- Intended to be executed from a master DbSchemaCLI script connected to the target Postgres DB (__DBSCHEMA_TARGET_SCHEMA__ schema). -- -- Note: This script intentionally does NOT delete internal migration/processing tables (mig_*, corp_processing, -- colin_tracking, affiliation_processing, etc). It only deletes the corp-scoped COLIN extract tables that are @@ -16,104 +16,104 @@ -- Address rows are treated as shared/global during subset refresh. -- Do not delete them here: subset_transfer_chunk.sql stages incoming Oracle address rows and --- merges them into public.address by addr_id. +-- merges them into the configured target address table by addr_id. -- Delete child tables first (event-scoped). -DELETE FROM notification_resend -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.notification_resend +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM notification -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.notification +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM filing_user -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.filing_user +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM payment -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.payment +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM ledger_text -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.ledger_text +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM conv_ledger -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.conv_ledger +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM conv_event -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.conv_event +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM completing_party -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.completing_party +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM submitting_party -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.submitting_party +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM corp_involved_amalgamating -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_involved_amalgamating +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM corp_involved_cont_in -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_involved_cont_in +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM correction -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.correction +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -DELETE FROM filing -WHERE event_id IN (SELECT event_id FROM event WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.filing +WHERE event_id IN (SELECT event_id FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in)); -- Delete corp-party related tables. -DELETE FROM party_notification -WHERE party_id IN (SELECT corp_party_id FROM corp_party WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.party_notification +WHERE party_id IN (SELECT corp_party_id FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party WHERE corp_num IN (&corp_ids_in)); -DELETE FROM offices_held -WHERE corp_party_id IN (SELECT corp_party_id FROM corp_party WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.offices_held +WHERE corp_party_id IN (SELECT corp_party_id FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party WHERE corp_num IN (&corp_ids_in)); -DELETE FROM corp_party_relationship -WHERE corp_party_id IN (SELECT corp_party_id FROM corp_party WHERE corp_num IN (&corp_ids_in)); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party_relationship +WHERE corp_party_id IN (SELECT corp_party_id FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party WHERE corp_num IN (&corp_ids_in)); -DELETE FROM corp_party +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party WHERE corp_num IN (&corp_ids_in); -- Delete corp-scoped tables. -DELETE FROM office +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.office WHERE corp_num IN (&corp_ids_in); -DELETE FROM corp_name +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_name WHERE corp_num IN (&corp_ids_in); -DELETE FROM corp_state +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_state WHERE corp_num IN (&corp_ids_in); -DELETE FROM corp_comments +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_comments WHERE corp_num IN (&corp_ids_in); -DELETE FROM corp_flag +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_flag WHERE corp_num IN (&corp_ids_in); -DELETE FROM cont_out +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.cont_out WHERE corp_num IN (&corp_ids_in); -DELETE FROM corp_restriction +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_restriction WHERE corp_num IN (&corp_ids_in); -DELETE FROM jurisdiction +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.jurisdiction WHERE corp_num IN (&corp_ids_in); -DELETE FROM resolution +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.resolution WHERE corp_num IN (&corp_ids_in); -DELETE FROM share_series +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.share_series WHERE corp_num IN (&corp_ids_in); -DELETE FROM share_struct_cls +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.share_struct_cls WHERE corp_num IN (&corp_ids_in); -DELETE FROM share_struct +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.share_struct WHERE corp_num IN (&corp_ids_in); -- Delete events last (many tables reference event_id). -DELETE FROM event +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.event WHERE corp_num IN (&corp_ids_in); -- Delete the corp rows last. -DELETE FROM corporation +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corporation WHERE corp_num IN (&corp_ids_in); -- Address rows are refreshed via stage+merge in subset_transfer_chunk.sql. diff --git a/data-tool/scripts/subset/subset_disable_triggers.sql b/data-tool/scripts/subset/subset_disable_triggers.sql index 6089f524c8..fd2c945fb3 100644 --- a/data-tool/scripts/subset/subset_disable_triggers.sql +++ b/data-tool/scripts/subset/subset_disable_triggers.sql @@ -1,38 +1,38 @@ -- Disable triggers for corp-scoped tables (subset refresh/load). -- Intended to be executed from a master DbSchemaCLI script while connected to the target Postgres DB. -ALTER TABLE corporation DISABLE TRIGGER ALL; -ALTER TABLE corp_name DISABLE TRIGGER ALL; -ALTER TABLE corp_state DISABLE TRIGGER ALL; -ALTER TABLE event DISABLE TRIGGER ALL; -ALTER TABLE filing DISABLE TRIGGER ALL; -ALTER TABLE filing_user DISABLE TRIGGER ALL; -ALTER TABLE office DISABLE TRIGGER ALL; -ALTER TABLE corp_comments DISABLE TRIGGER ALL; -ALTER TABLE ledger_text DISABLE TRIGGER ALL; -ALTER TABLE corp_party DISABLE TRIGGER ALL; -ALTER TABLE corp_party_relationship DISABLE TRIGGER ALL; -ALTER TABLE offices_held DISABLE TRIGGER ALL; -ALTER TABLE completing_party DISABLE TRIGGER ALL; -ALTER TABLE submitting_party DISABLE TRIGGER ALL; -ALTER TABLE corp_flag DISABLE TRIGGER ALL; -ALTER TABLE cont_out DISABLE TRIGGER ALL; -ALTER TABLE conv_event DISABLE TRIGGER ALL; -ALTER TABLE conv_ledger DISABLE TRIGGER ALL; -ALTER TABLE corp_involved_amalgamating DISABLE TRIGGER ALL; -ALTER TABLE corp_involved_cont_in DISABLE TRIGGER ALL; -ALTER TABLE corp_restriction DISABLE TRIGGER ALL; -ALTER TABLE correction DISABLE TRIGGER ALL; -ALTER TABLE jurisdiction DISABLE TRIGGER ALL; -ALTER TABLE resolution DISABLE TRIGGER ALL; -ALTER TABLE share_series DISABLE TRIGGER ALL; -ALTER TABLE share_struct DISABLE TRIGGER ALL; -ALTER TABLE share_struct_cls DISABLE TRIGGER ALL; -ALTER TABLE notification DISABLE TRIGGER ALL; -ALTER TABLE notification_resend DISABLE TRIGGER ALL; -ALTER TABLE party_notification DISABLE TRIGGER ALL; -ALTER TABLE payment DISABLE TRIGGER ALL; -ALTER TABLE carsfile DISABLE TRIGGER ALL; -ALTER TABLE carsbox DISABLE TRIGGER ALL; -ALTER TABLE carsrept DISABLE TRIGGER ALL; -ALTER TABLE carindiv DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corporation DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_name DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_state DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.event DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.filing DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.filing_user DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.office DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_comments DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.ledger_text DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_party DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_party_relationship DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.offices_held DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.completing_party DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.submitting_party DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_flag DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.cont_out DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.conv_event DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.conv_ledger DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_involved_amalgamating DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_involved_cont_in DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_restriction DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.correction DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.jurisdiction DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.resolution DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.share_series DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.share_struct DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.share_struct_cls DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.notification DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.notification_resend DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.party_notification DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.payment DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.carsfile DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.carsbox DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.carsrept DISABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.carindiv DISABLE TRIGGER ALL; diff --git a/data-tool/scripts/subset/subset_enable_triggers.sql b/data-tool/scripts/subset/subset_enable_triggers.sql index 071691976d..b7ef1327ac 100644 --- a/data-tool/scripts/subset/subset_enable_triggers.sql +++ b/data-tool/scripts/subset/subset_enable_triggers.sql @@ -1,38 +1,38 @@ -- Enable triggers for corp-scoped tables (subset refresh/load). -- Intended to be executed from a master DbSchemaCLI script while connected to the target Postgres DB. -ALTER TABLE corporation ENABLE TRIGGER ALL; -ALTER TABLE corp_name ENABLE TRIGGER ALL; -ALTER TABLE corp_state ENABLE TRIGGER ALL; -ALTER TABLE event ENABLE TRIGGER ALL; -ALTER TABLE filing ENABLE TRIGGER ALL; -ALTER TABLE filing_user ENABLE TRIGGER ALL; -ALTER TABLE office ENABLE TRIGGER ALL; -ALTER TABLE corp_comments ENABLE TRIGGER ALL; -ALTER TABLE ledger_text ENABLE TRIGGER ALL; -ALTER TABLE corp_party ENABLE TRIGGER ALL; -ALTER TABLE corp_party_relationship ENABLE TRIGGER ALL; -ALTER TABLE offices_held ENABLE TRIGGER ALL; -ALTER TABLE completing_party ENABLE TRIGGER ALL; -ALTER TABLE submitting_party ENABLE TRIGGER ALL; -ALTER TABLE corp_flag ENABLE TRIGGER ALL; -ALTER TABLE cont_out ENABLE TRIGGER ALL; -ALTER TABLE conv_event ENABLE TRIGGER ALL; -ALTER TABLE conv_ledger ENABLE TRIGGER ALL; -ALTER TABLE corp_involved_amalgamating ENABLE TRIGGER ALL; -ALTER TABLE corp_involved_cont_in ENABLE TRIGGER ALL; -ALTER TABLE corp_restriction ENABLE TRIGGER ALL; -ALTER TABLE correction ENABLE TRIGGER ALL; -ALTER TABLE jurisdiction ENABLE TRIGGER ALL; -ALTER TABLE resolution ENABLE TRIGGER ALL; -ALTER TABLE share_series ENABLE TRIGGER ALL; -ALTER TABLE share_struct ENABLE TRIGGER ALL; -ALTER TABLE share_struct_cls ENABLE TRIGGER ALL; -ALTER TABLE notification ENABLE TRIGGER ALL; -ALTER TABLE notification_resend ENABLE TRIGGER ALL; -ALTER TABLE party_notification ENABLE TRIGGER ALL; -ALTER TABLE payment ENABLE TRIGGER ALL; -ALTER TABLE carsfile ENABLE TRIGGER ALL; -ALTER TABLE carsbox ENABLE TRIGGER ALL; -ALTER TABLE carsrept ENABLE TRIGGER ALL; -ALTER TABLE carindiv ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corporation ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_name ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_state ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.event ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.filing ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.filing_user ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.office ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_comments ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.ledger_text ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_party ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_party_relationship ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.offices_held ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.completing_party ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.submitting_party ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_flag ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.cont_out ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.conv_event ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.conv_ledger ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_involved_amalgamating ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_involved_cont_in ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.corp_restriction ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.correction ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.jurisdiction ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.resolution ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.share_series ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.share_struct ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.share_struct_cls ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.notification ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.notification_resend ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.party_notification ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.payment ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.carsfile ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.carsbox ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.carsrept ENABLE TRIGGER ALL; +ALTER TABLE __DBSCHEMA_TARGET_SCHEMA__.carindiv ENABLE TRIGGER ALL; diff --git a/data-tool/scripts/subset/subset_pg_boolean_casts.sql b/data-tool/scripts/subset/subset_pg_boolean_casts.sql index b97a72568d..b141cd5e00 100644 --- a/data-tool/scripts/subset/subset_pg_boolean_casts.sql +++ b/data-tool/scripts/subset/subset_pg_boolean_casts.sql @@ -23,7 +23,7 @@ -- DbSchemaCLI splits statements on semicolons and does not reliably handle semicolons inside dollar-quoted -- bodies. Keep dollar-quoted bodies free of internal semicolons and avoid DO $$ blocks. -CREATE OR REPLACE FUNCTION public.dbcli_varchar_to_boolean(val varchar) +CREATE OR REPLACE FUNCTION __DBSCHEMA_TARGET_SCHEMA__.dbcli_varchar_to_boolean(val varchar) RETURNS boolean LANGUAGE sql IMMUTABLE @@ -32,7 +32,7 @@ AS $$ SELECT (val::text)::boolean $$; -CREATE OR REPLACE FUNCTION public.dbcli_bpchar_to_boolean(val bpchar) +CREATE OR REPLACE FUNCTION __DBSCHEMA_TARGET_SCHEMA__.dbcli_bpchar_to_boolean(val bpchar) RETURNS boolean LANGUAGE sql IMMUTABLE @@ -44,12 +44,12 @@ $$; -- Recreate casts in an idempotent way (Postgres has no CREATE CAST IF NOT EXISTS). DROP CAST IF EXISTS (varchar AS boolean); CREATE CAST (varchar AS boolean) - WITH FUNCTION public.dbcli_varchar_to_boolean(varchar) + WITH FUNCTION __DBSCHEMA_TARGET_SCHEMA__.dbcli_varchar_to_boolean(varchar) AS IMPLICIT -- DbSchemaCLI workaround: avoid keyword being last token ; DROP CAST IF EXISTS (bpchar AS boolean); CREATE CAST (bpchar AS boolean) - WITH FUNCTION public.dbcli_bpchar_to_boolean(bpchar) + WITH FUNCTION __DBSCHEMA_TARGET_SCHEMA__.dbcli_bpchar_to_boolean(bpchar) AS IMPLICIT -- DbSchemaCLI workaround: avoid keyword being last token ; diff --git a/data-tool/scripts/subset/subset_pg_cleanup_address_stage.sql b/data-tool/scripts/subset/subset_pg_cleanup_address_stage.sql index a702de8e9c..407d969791 100644 --- a/data-tool/scripts/subset/subset_pg_cleanup_address_stage.sql +++ b/data-tool/scripts/subset/subset_pg_cleanup_address_stage.sql @@ -1,2 +1,4 @@ -- Cleanup the shared address staging table used by subset_transfer_chunk.sql. -- No-op: the helper table is predeclared in the COLIN extract DDL and is truncated during prepare/chunk execution. + +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.subset_address_stage; diff --git a/data-tool/scripts/subset/subset_pg_cleanup_orphan_children.sql b/data-tool/scripts/subset/subset_pg_cleanup_orphan_children.sql index 345a6e55e1..33199b4f71 100644 --- a/data-tool/scripts/subset/subset_pg_cleanup_orphan_children.sql +++ b/data-tool/scripts/subset/subset_pg_cleanup_orphan_children.sql @@ -11,52 +11,52 @@ -- - corp-scoped rows deleted directly by corp_num are left to the regular chunk deletes -- Event-scoped children whose parent event row is missing. -DELETE FROM notification_resend t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.notification_resend t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM notification t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.notification t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM filing_user t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.filing_user t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM payment t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.payment t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM ledger_text t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.ledger_text t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM conv_ledger t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.conv_ledger t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM conv_event t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.conv_event t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM completing_party t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.completing_party t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM submitting_party t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.submitting_party t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM corp_involved_amalgamating t +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_involved_amalgamating t WHERE t.event_id IS NOT NULL - AND NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); + AND NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM corp_involved_cont_in t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_involved_cont_in t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM correction t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.correction t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -DELETE FROM filing t -WHERE NOT EXISTS (SELECT 1 FROM event e WHERE e.event_id = t.event_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.filing t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.event e WHERE e.event_id = t.event_id); -- Corp-party children whose parent corp_party row is missing. -DELETE FROM party_notification t -WHERE NOT EXISTS (SELECT 1 FROM corp_party cp WHERE cp.corp_party_id = t.party_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.party_notification t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party cp WHERE cp.corp_party_id = t.party_id); -DELETE FROM offices_held t -WHERE NOT EXISTS (SELECT 1 FROM corp_party cp WHERE cp.corp_party_id = t.corp_party_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.offices_held t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party cp WHERE cp.corp_party_id = t.corp_party_id); -DELETE FROM corp_party_relationship t -WHERE NOT EXISTS (SELECT 1 FROM corp_party cp WHERE cp.corp_party_id = t.corp_party_id); +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party_relationship t +WHERE NOT EXISTS (SELECT 1 FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party cp WHERE cp.corp_party_id = t.corp_party_id); diff --git a/data-tool/scripts/subset/subset_pg_prepare_address_stage.sql b/data-tool/scripts/subset/subset_pg_prepare_address_stage.sql index 12c00c30d3..0305556ec2 100644 --- a/data-tool/scripts/subset/subset_pg_prepare_address_stage.sql +++ b/data-tool/scripts/subset/subset_pg_prepare_address_stage.sql @@ -1,4 +1,4 @@ -- Prepare the shared address staging table used by subset_transfer_chunk.sql. -- This is a predeclared regular table (not TEMP) because DbSchemaCLI transfer work may use separate sessions. -TRUNCATE TABLE public.subset_address_stage; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.subset_address_stage; diff --git a/data-tool/scripts/subset/subset_pg_purge_bcomps_excluded.sql b/data-tool/scripts/subset/subset_pg_purge_bcomps_excluded.sql index 7c48f0e562..e7313ed0dd 100644 --- a/data-tool/scripts/subset/subset_pg_purge_bcomps_excluded.sql +++ b/data-tool/scripts/subset/subset_pg_purge_bcomps_excluded.sql @@ -15,167 +15,167 @@ -- 1) Build keysets -TRUNCATE TABLE public.subset_excluded_corp_parties; -TRUNCATE TABLE public.subset_excluded_events; -TRUNCATE TABLE public.subset_excluded_corps; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corp_parties; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps; -INSERT INTO public.subset_excluded_corps (corp_num) +INSERT INTO __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps (corp_num) SELECT DISTINCT e.corp_num -FROM event e -JOIN filing f ON f.event_id = e.event_id -JOIN filing_user u ON u.event_id = e.event_id +FROM __DBSCHEMA_TARGET_SCHEMA__.event e +JOIN __DBSCHEMA_TARGET_SCHEMA__.filing f ON f.event_id = e.event_id +JOIN __DBSCHEMA_TARGET_SCHEMA__.filing_user u ON u.event_id = e.event_id WHERE e.corp_num IS NOT NULL AND u.user_id = 'BCOMPS' AND f.filing_type_cd IN ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC'); -INSERT INTO public.subset_excluded_events (event_id) +INSERT INTO __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events (event_id) SELECT DISTINCT e.event_id -FROM event e -JOIN public.subset_excluded_corps x ON x.corp_num = e.corp_num +FROM __DBSCHEMA_TARGET_SCHEMA__.event e +JOIN __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x ON x.corp_num = e.corp_num WHERE e.event_id IS NOT NULL; -INSERT INTO public.subset_excluded_corp_parties (corp_party_id) +INSERT INTO __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corp_parties (corp_party_id) SELECT DISTINCT cp.corp_party_id -FROM corp_party cp -JOIN public.subset_excluded_corps x ON x.corp_num = cp.corp_num +FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party cp +JOIN __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x ON x.corp_num = cp.corp_num WHERE cp.corp_party_id IS NOT NULL; -- 2) Purge (delete child tables first) -- Event-scoped children -DELETE FROM notification_resend t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.notification_resend t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -DELETE FROM notification t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.notification t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -DELETE FROM filing_user t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.filing_user t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -DELETE FROM payment t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.payment t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -DELETE FROM ledger_text t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.ledger_text t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -DELETE FROM conv_ledger t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.conv_ledger t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -DELETE FROM conv_event t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.conv_event t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -DELETE FROM completing_party t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.completing_party t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -DELETE FROM submitting_party t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.submitting_party t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -DELETE FROM corp_involved_cont_in t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_involved_cont_in t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -DELETE FROM correction t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.correction t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -DELETE FROM filing t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.filing t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -- corp_involved_amalgamating can reference corp_num via ted_corp_num/ting_corp_num as well as event_id. -- Delete any rows where either side is excluded (covers non-event-owned references too). -DELETE FROM corp_involved_amalgamating t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_involved_amalgamating t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.ted_corp_num = x.corp_num OR t.ting_corp_num = x.corp_num; -- Corp-party related -DELETE FROM party_notification t -USING public.subset_excluded_corp_parties x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.party_notification t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corp_parties x WHERE t.party_id = x.corp_party_id; -DELETE FROM offices_held t -USING public.subset_excluded_corp_parties x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.offices_held t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corp_parties x WHERE t.corp_party_id = x.corp_party_id; -DELETE FROM corp_party_relationship t -USING public.subset_excluded_corp_parties x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party_relationship t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corp_parties x WHERE t.corp_party_id = x.corp_party_id; -DELETE FROM corp_party t -USING public.subset_excluded_corp_parties x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_party t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corp_parties x WHERE t.corp_party_id = x.corp_party_id; -- Corp-scoped tables -DELETE FROM office t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.office t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -DELETE FROM corp_name t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_name t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -DELETE FROM corp_state t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_state t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -DELETE FROM corp_comments t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_comments t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -DELETE FROM corp_flag t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_flag t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -DELETE FROM cont_out t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.cont_out t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -DELETE FROM corp_restriction t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corp_restriction t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -DELETE FROM jurisdiction t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.jurisdiction t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -DELETE FROM resolution t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.resolution t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -- Share tables (delete deepest first) -DELETE FROM share_series t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.share_series t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -DELETE FROM share_struct_cls t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.share_struct_cls t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -DELETE FROM share_struct t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.share_struct t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -- Events last (many tables reference event_id) -DELETE FROM event t -USING public.subset_excluded_events x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.event t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events x WHERE t.event_id = x.event_id; -- Corporation last -DELETE FROM corporation t -USING public.subset_excluded_corps x +DELETE FROM __DBSCHEMA_TARGET_SCHEMA__.corporation t +USING __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps x WHERE t.corp_num = x.corp_num; -- 3) Cleanup helper tables -TRUNCATE TABLE public.subset_excluded_corp_parties; -TRUNCATE TABLE public.subset_excluded_events; -TRUNCATE TABLE public.subset_excluded_corps; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corp_parties; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_events; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.subset_excluded_corps; diff --git a/data-tool/scripts/subset/subset_transfer_cars.sql b/data-tool/scripts/subset/subset_transfer_cars.sql index f41876127a..665db97810 100644 --- a/data-tool/scripts/subset/subset_transfer_cars.sql +++ b/data-tool/scripts/subset/subset_transfer_cars.sql @@ -1,17 +1,17 @@ --- Global transfer of cars* tables from SOURCE Oracle DB (cprd) into TARGET Postgres extract DB (cprd_pg). --- Intended to be executed from a master DbSchemaCLI script connected to the target Postgres DB (cprd_pg). +-- Global transfer of cars* tables from SOURCE Oracle DB (__DBSCHEMA_SOURCE_CONNECTION__) into TARGET Postgres extract DB (__DBSCHEMA_TARGET_SCHEMA__ schema). +-- Intended to be executed from a master DbSchemaCLI script connected to the target Postgres DB (__DBSCHEMA_TARGET_SCHEMA__ schema). -- -- These tables are NOT corp-scoped. The full dataset is transferred without filtering. -- Volume is low enough that a full refresh is appropriate. -transfer public.carsfile from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.carsfile from __DBSCHEMA_SOURCE_CONNECTION__ using select documtid, filedate, regiracf from carsfile; -transfer public.carsbox from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.carsbox from __DBSCHEMA_SOURCE_CONNECTION__ using select documtid, accesnum, @@ -19,14 +19,14 @@ select boxrracf from carsbox; -transfer public.carsrept from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.carsrept from __DBSCHEMA_SOURCE_CONNECTION__ using select documtid, docutype, compnumb from carsrept; -transfer public.carindiv from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.carindiv from __DBSCHEMA_SOURCE_CONNECTION__ using select documtid, replace(surname, CHR(0), '') as surname, diff --git a/data-tool/scripts/subset/subset_transfer_chunk.sql b/data-tool/scripts/subset/subset_transfer_chunk.sql index 50b4e91757..f76703338c 100644 --- a/data-tool/scripts/subset/subset_transfer_chunk.sql +++ b/data-tool/scripts/subset/subset_transfer_chunk.sql @@ -1,4 +1,4 @@ --- Transfer a chunk (or a whole subset) of corps from the SOURCE Oracle DB (cprd) into the TARGET Postgres extract DB (cprd_pg). +-- Transfer a chunk (or a whole subset) of corps from the SOURCE Oracle DB (__DBSCHEMA_SOURCE_CONNECTION__) into the TARGET Postgres extract DB (__DBSCHEMA_TARGET_SCHEMA__ schema). -- -- REQUIRED DbSchemaCLI variables (replace_variables=true): -- target_corp_num_predicate : SQL predicate restricting the computed target_corp_num (NO trailing semicolon). @@ -14,7 +14,7 @@ -- c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE') -- c.CORP_TYP_CD in ('BC','C','ULC','CUL','CC','CCC','QA','QB','QC','QD','QE','CP') -- --- Intended to be executed from a master DbSchemaCLI script connected to the target Postgres DB (cprd_pg). +-- Intended to be executed from a master DbSchemaCLI script connected to the target Postgres DB (__DBSCHEMA_TARGET_SCHEMA__ schema). -- -- IMPORTANT: -- - This template intentionally avoids the boolean<->integer ALTER COLUMN hacks used in the full refresh script. @@ -33,7 +33,7 @@ -- vset oracle_corp_num_predicate=c.CORP_NUM in ('1111585','1226175'); -- corporation -transfer public.corporation from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.corporation from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -88,7 +88,7 @@ left join last_ar la on la.corp_num = c.corp_num; -- event -transfer public.event from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.event from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -121,7 +121,7 @@ where e.event_typ_cd not in ('BNUPD', 'ADDLEDGR'); -- corp_name -transfer public.corp_name from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.corp_name from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -152,7 +152,7 @@ join CORP_NAME cn on cn.corp_num = c.corp_num; -- corp_state -transfer public.corp_state from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.corp_state from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -184,7 +184,7 @@ join corp_op_state cos on cos.state_typ_cd = cs.state_typ_cd; -- filing -transfer public.filing from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.filing from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -231,7 +231,7 @@ join filing f on f.event_id = e.event_id; -- filing_user -transfer public.filing_user from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.filing_user from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -266,9 +266,9 @@ join filing_user u on u.event_id = e.event_id; -- address (shared/global table; stage then merge before loading dependents) -TRUNCATE TABLE public.subset_address_stage; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.subset_address_stage; -transfer public.subset_address_stage from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.subset_address_stage from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -346,7 +346,7 @@ from ( join address a on x.mailing_addr_id = a.addr_id ); -INSERT INTO public.address ( +INSERT INTO __DBSCHEMA_TARGET_SCHEMA__.address ( addr_id, province, country_typ_cd, @@ -374,7 +374,7 @@ FROM ( addr_line_2, addr_line_3, city - FROM public.subset_address_stage + FROM __DBSCHEMA_TARGET_SCHEMA__.subset_address_stage WHERE addr_id IS NOT NULL ORDER BY addr_id ) s @@ -387,11 +387,11 @@ SET province = EXCLUDED.province, addr_line_3 = EXCLUDED.addr_line_3, city = EXCLUDED.city; -TRUNCATE TABLE public.subset_address_stage; +TRUNCATE TABLE __DBSCHEMA_TARGET_SCHEMA__.subset_address_stage; -- office -transfer public.office from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.office from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -423,7 +423,7 @@ join office o on o.corp_num = c.corp_num; -- corp_comments -transfer public.corp_comments from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.corp_comments from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -457,7 +457,7 @@ join corp_comments cc on cc.corp_num = c.corp_num; -- ledger_text -transfer public.ledger_text from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.ledger_text from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -488,7 +488,7 @@ join ledger_text l on l.event_id = e.event_id; -- corp_party -transfer public.corp_party from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.corp_party from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -531,7 +531,7 @@ join corp_party p on p.corp_num = c.corp_num; -- corp_party_relationship -transfer public.corp_party_relationship from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.corp_party_relationship from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -560,7 +560,7 @@ join CORP_PARTY_RELATIONSHIP cpr on cpr.corp_party_id = p.corp_party_id; -- offices_held -transfer public.offices_held from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.offices_held from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -589,7 +589,7 @@ join OFFICES_HELD oh on oh.corp_party_id = p.corp_party_id; -- completing_party -transfer public.completing_party from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.completing_party from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -622,7 +622,7 @@ join completing_party cp on cp.event_id = e.event_id; -- submitting_party -transfer public.submitting_party from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.submitting_party from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -663,7 +663,7 @@ join SUBMITTING_PARTY sp on sp.event_id = e.event_id; -- corp_flag -transfer public.corp_flag from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.corp_flag from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -693,7 +693,7 @@ join corp_flag cf on cf.corp_num = c.corp_num; -- cont_out -transfer public.cont_out from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.cont_out from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -726,7 +726,7 @@ join CONT_OUT co on co.corp_num = c.corp_num; -- conv_event -transfer public.conv_event from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.conv_event from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -765,7 +765,7 @@ join CONV_EVENT ce on ce.event_id = e.event_id; -- conv_ledger -transfer public.conv_ledger from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.conv_ledger from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -796,7 +796,7 @@ join CONV_LEDGER cl on cl.event_id = e.event_id; -- corp_involved - amalgamaTING_businesses -transfer public.corp_involved_amalgamating from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.corp_involved_amalgamating from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -864,7 +864,7 @@ where f.filing_typ_cd in ('AMALH', 'AMALV', 'AMALR', 'AMLHU', 'AMLVU', 'AMLRU', -- corp_involved - continue_in_historical_xpro -transfer public.corp_involved_cont_in from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.corp_involved_cont_in from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -895,7 +895,7 @@ where f.filing_typ_cd in ('CONTI', 'CONTU', 'CONTC') -- corp_restriction -transfer public.corp_restriction from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.corp_restriction from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -929,7 +929,7 @@ join CORP_RESTRICTION cr on cr.corp_num = c.corp_num; -- correction -transfer public.correction from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.correction from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -959,7 +959,7 @@ join CORRECTION corr on corr.event_id = e.event_id; -- continued_in_from_jurisdiction -transfer public.jurisdiction from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.jurisdiction from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -994,7 +994,7 @@ join JURISDICTION j on j.corp_num = c.corp_num; -- resolution -transfer public.resolution from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.resolution from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -1025,7 +1025,7 @@ join RESOLUTION r on r.corp_num = c.corp_num; -- share_struct -transfer public.share_struct from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.share_struct from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -1054,7 +1054,7 @@ join SHARE_STRUCT ss on ss.corp_num = c.corp_num; -- share_struct_cls -transfer public.share_struct_cls from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.share_struct_cls from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -1103,7 +1103,7 @@ join SHARE_STRUCT_CLS ssc on ssc.corp_num = c.corp_num; -- share_series -transfer public.share_series from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.share_series from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -1145,7 +1145,7 @@ join SHARE_SERIES ss on ss.corp_num = c.corp_num; -- notification -transfer public.notification from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.notification from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -1181,7 +1181,7 @@ join NOTIFICATION n on n.event_id = e.event_id; -- notification_resend -transfer public.notification_resend from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.notification_resend from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -1217,7 +1217,7 @@ join NOTIFICATION_RESEND nr on nr.event_id = e.event_id; -- party_notification -transfer public.party_notification from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.party_notification from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -1254,7 +1254,7 @@ join PARTY_NOTIFICATION pn on pn.party_id = cp.corp_party_id; -- payment -transfer public.payment from cprd using +transfer __DBSCHEMA_TARGET_SCHEMA__.payment from __DBSCHEMA_SOURCE_CONNECTION__ using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c