diff --git a/app/app.py b/app/app.py index 6ae4e5b..74d8e3c 100644 --- a/app/app.py +++ b/app/app.py @@ -46,6 +46,12 @@ def __init__(self, **kwargs): } DEFAULT_MODE = "categories" BINDINGS = [ + Binding( + "q", + "quit", + "Quit", + tooltip="Quit the application", + ), Binding( "r", "app.switch_mode('readme')", diff --git a/app/executor.py b/app/executor.py index 52f0243..26cae73 100644 --- a/app/executor.py +++ b/app/executor.py @@ -4,7 +4,7 @@ from textual.binding import Binding from textual.app import ComposeResult from textual.containers import Grid, Center, Vertical, VerticalScroll, ScrollableContainer -from textual.widgets import Footer, Label, Markdown, Static, Input, Select, RichLog, TabbedContent +from textual.widgets import Footer, Label, Markdown, Static, Input, Select, RichLog, TabbedContent, SelectionList from textual.screen import Screen, ModalScreen from app.header import MCHeader from textual import events, on, work @@ -43,6 +43,11 @@ class ExecutorArgs(Vertical, can_focus=False, can_focus_children=True): padding: 1; text-align: right; } + SelectionList { + height: 6; + min-height: 6; + max-height: 6; + } } } """ @@ -78,12 +83,15 @@ def compose(self) -> ComposeResult: yield Label(f"{mandatory}{arg}") if arg == 'profile': yield Select((profile, profile) for profile in self.get_profiles()) + elif arg == 'entities' and arguments[arg].get('choices'): + # Multi-select for entities + choices = arguments[arg]['choices'] + yield SelectionList(*[(choice, choice, choice == 'all') for choice in choices], id='entities') + elif arguments[arg].get('choices'): + choices = arguments[arg]['choices'] + yield Select((choice, choice) for choice in choices) else: - if arguments[arg].get('choices'): - choices = arguments[arg]['choices'] - yield Select((choice, choice) for choice in choices) - else: - yield Input(placeholder=arguments[arg]['help']) + yield Input(placeholder=arguments[arg]['help']) def get_profiles(self): """Read headers from mcd profiles.ini""" @@ -116,13 +124,16 @@ def update_grid_with_selected_subparser(self) -> None: grid._add_child(Label(f"{mandatory}{arg}")) if arg == 'profile': grid._add_child(Select((profile, profile) for profile in self.get_profiles())) + elif arg == 'entities' and selected_subparser_arguments[arg].get('choices'): + # Multi-select for entities + choices = selected_subparser_arguments[arg]['choices'] + grid._add_child(SelectionList(*[(choice, choice, choice == 'all') for choice in choices], id='entities')) + elif selected_subparser_arguments[arg].get('choices'): + choices = selected_subparser_arguments[arg]['choices'] + grid._add_child(Select.from_values(choices)) else: - if selected_subparser_arguments[arg].get('choices'): - choices = selected_subparser_arguments[arg]['choices'] - grid._add_child(Select.from_values(choices)) - else: - grid._add_child(Input(placeholder=selected_subparser_arguments[arg]['help'], - tooltip=selected_subparser_arguments[arg]['help'])) + grid._add_child(Input(placeholder=selected_subparser_arguments[arg]['help'], + tooltip=selected_subparser_arguments[arg]['help'])) nodes = grid.query_children().nodes grid.mount(*nodes) @@ -173,7 +184,13 @@ async def action_run_utility(self): args.append(f"-{flag[0]}") if '★' in element._Static__content: mandatory = True - if isinstance(element, Input) or isinstance(element, Select): + if isinstance(element, SelectionList): + # Multi-select: join selected values with commas + selected = list(element.selected) + value = ','.join(selected) if selected else 'all' + args.append(value) + mandatory = False + elif isinstance(element, Input) or isinstance(element, Select): value = element.value if mandatory and value == '': self.notify(f"'{flags[-1]}' is required", severity="error") diff --git a/lib/helpers/parser_config.json b/lib/helpers/parser_config.json index 467d0d1..cd72630 100644 --- a/lib/helpers/parser_config.json +++ b/lib/helpers/parser_config.json @@ -590,10 +590,10 @@ }, "migration": { "workspace_migrator.py": { - "description": "\nWorkspace migration utility for Monte Carlo configurations.\n\nExport, import, and validate domains, data products, and blocklists between environments.\n\nExamples:\n\t# Export all entities\n\tpython migration/workspace_migrator.py export --profile source_env\n\n\t# Export specific entities\n\tpython migration/workspace_migrator.py export --profile source_env --entities domains,blocklists\n\n\t# Import (dry-run by default)\n\tpython migration/workspace_migrator.py import --profile target_env\n\n\t# Import with force (commit changes)\n\tpython migration/workspace_migrator.py import --profile target_env --force yes\n\n\t# Validate migration files\n\tpython migration/workspace_migrator.py validate --profile target_env", + "description": "\nWorkspace migration utility for Monte Carlo configurations.\n\nExport, import, and validate all MC configurations between environments. Supports: blocklists, domains, tags, exclusion_windows, data_products, audiences, and monitors.\n\nExamples:\n\t# Export all entities\n\tpython migration/workspace_migrator.py export --profile source_env\n\n\t# Export specific entities\n\tpython migration/workspace_migrator.py export --profile source_env --entities domains,blocklists,tags\n\n\t# Import (dry-run by default)\n\tpython migration/workspace_migrator.py import --profile target_env\n\n\t# Import with force (commit changes)\n\tpython migration/workspace_migrator.py import --profile target_env --force yes\n\n\t# Validate migration files\n\tpython migration/workspace_migrator.py validate --profile target_env", "subparsers": { "export": { - "description": "Export domains, data products, and blocklists to CSV files.", + "description": "Export MC configurations to files. Exports blocklists, domains, tags, exclusion_windows, data_products, and audiences to CSV files. Exports monitors to YAML (MaC format).\n\nExample:\n\tpython migration/workspace_migrator.py export --profile source_env --entities all", "help": "Export MC configurations to files.", "arguments": { "profile": { @@ -604,16 +604,17 @@ "entities": { "required": false, "default": "all", - "help": "Comma-separated list of entities: blocklists,domains,tags,exclusion_windows,data_products,monitors (or 'all')" + "choices": ["all", "blocklists", "domains", "tags", "exclusion_windows", "data_products", "audiences", "monitors"], + "help": "Comma-separated list of entity types to export, or 'all' for all entities. Example: 'domains,blocklists,tags'. Entities are exported in dependency order." }, "output_dir": { "required": false, - "help": "Directory to write export files. Defaults to migration/migration-data-exports/" + "help": "Directory to write export files. Default: migration/migration-data-exports/" } } }, "import": { - "description": "Import domains, data products, blocklists, and monitors from files. Dry-run by default.", + "description": "Import MC configurations from files. Imports blocklists, domains, tags, exclusion_windows, data_products, and audiences from CSV files. Imports monitors from YAML (MaC format). Dry-run by default.\n\nEntities are imported in dependency order: blocklists → domains → tags → exclusion_windows → data_products → audiences → monitors.\n\nWarehouse mapping is required for tags and monitors when warehouse names differ between source and target environments.\n\nFor monitors: Monitors imported via MaC are code-managed by default. Use --convert_to_ui flag to auto-convert after import, or use the 'convert-to-ui' subcommand separately after import.\n\nExample:\n\tpython migration/workspace_migrator.py import --profile target_env --entities tags,monitors --warehouse_map \"US Warehouse=EU Warehouse\" --convert_to_ui --force yes", "help": "Import MC configurations from files.", "arguments": { "profile": { @@ -624,70 +625,72 @@ "entities": { "required": false, "default": "all", - "help": "Comma-separated list of entities: blocklists,domains,tags,exclusion_windows,data_products,monitors (or 'all')" + "choices": ["all", "blocklists", "domains", "tags", "exclusion_windows", "data_products", "audiences", "monitors"], + "help": "Comma-separated list of entity types to import, or 'all' for all entities. Example: 'domains,blocklists,tags'. Entities are imported in dependency order." }, "input_dir": { "required": false, - "help": "Directory containing import files. Defaults to migration/migration-data-exports/" + "help": "Directory containing import files. Default: migration/migration-data-exports/" }, "warehouse_map": { "required": false, - "help": "Warehouse name mapping for tag migrations: 'Source1=Dest1,Source2=Dest2'. Or create warehouse_mapping.json in input_dir." + "help": "Warehouse name mapping for tags and monitors migrations. Required when warehouse names differ between source and target environments.\n\nFormat: 'SourceName1=DestName1,SourceName2=DestName2'\nExample: 'US Snowflake=EU Snowflake,US BigQuery=EU BigQuery'\n\nAlternatively, create warehouse_mapping.json in input_dir with format:\n{\n \"warehouse_mapping\": {\n \"Source Name\": \"Destination Name\"\n }\n}" }, - "force": { - "required": false, - "choices": ["yes", "no"], - "help": "Set to 'yes' to commit changes. Default is dry-run (preview only)." - }, - "convert_to_ui": { - "required": false, - "action": "store_true", - "help": "Convert monitors to UI-editable after import." + "force": { + "required": false, + "choices": ["yes", "no"], + "help": "Set to 'yes' to commit changes. Default is dry-run (preview only). Dry-run shows what would be created/updated/skipped without making changes." + }, + "convert_to_ui": { + "required": false, + "action": "store_true", + "help": "Convert monitors to UI-editable after import. Monitors imported via MaC are code-managed by default and cannot be edited in the UI. This flag converts them to UI-editable monitors." + } } - } - }, - "validate": { - "description": "Validate migration files before importing.", - "help": "Validate migration files.", - "arguments": { - "profile": { - "default": "default", - "required": false, - "help": "Specify an MCD profile name. Uses default otherwise." - }, - "entities": { - "required": false, - "default": "all", - "help": "Comma-separated list of entities: blocklists,domains,tags,exclusion_windows,data_products,monitors (or 'all')" - }, - "input_dir": { - "required": false, - "help": "Directory containing files to validate. Defaults to migration/migration-data-exports/" + }, + "validate": { + "description": "Validate migration files before importing. Checks file format, required fields, and data integrity.\n\nExample:\n\tpython migration/workspace_migrator.py validate --profile target_env --entities all", + "help": "Validate migration files before importing.", + "arguments": { + "profile": { + "default": "default", + "required": false, + "help": "Specify an MCD profile name. Uses default otherwise." + }, + "entities": { + "required": false, + "default": "all", + "choices": ["all", "blocklists", "domains", "tags", "exclusion_windows", "data_products", "audiences", "monitors"], + "help": "Comma-separated list of entity types to validate, or 'all' for all entities. Example: 'domains,blocklists,tags'" + }, + "input_dir": { + "required": false, + "help": "Directory containing files to validate. Default: migration/migration-data-exports/" + } } - } - }, - "convert-to-ui": { - "description": "Convert monitors in a namespace from code-deployed to UI-editable.", - "help": "Convert monitors to UI-editable.", - "arguments": { - "profile": { - "default": "default", - "required": false, - "help": "Specify an MCD profile name. Uses default otherwise." - }, - "namespace": { - "required": false, - "default": "migration", - "help": "Namespace containing monitors to convert. Defaults to 'migration'." - }, - "force": { - "required": false, - "choices": ["yes", "no"], - "help": "Set to 'yes' to commit changes. Default is dry-run (preview only)." + }, + "convert-to-ui": { + "description": "Convert monitors in a namespace from code-deployed to UI-editable. Monitors imported via MaC (Monitors as Code) are code-managed by default and cannot be edited in the Monte Carlo UI. This command converts them to UI-editable monitors.\n\nUse this subcommand if you imported monitors without the --convert_to_ui flag, or if you want to convert monitors from a previous import.\n\nAfter conversion, monitors move from the custom namespace (e.g., 'migration') to the 'ui' namespace.\n\nExample:\n\tpython migration/workspace_migrator.py convert-to-ui --profile target_env --namespace migration --force yes", + "help": "Convert monitors to UI-editable.", + "arguments": { + "profile": { + "default": "default", + "required": false, + "help": "Specify an MCD profile name. Uses default otherwise." + }, + "namespace": { + "required": false, + "default": "migration", + "help": "Namespace containing monitors to convert. Defaults to 'migration'. Monitors imported via the migration tool are placed in this namespace by default." + }, + "force": { + "required": false, + "choices": ["yes", "no"], + "help": "Set to 'yes' to commit changes. Default is dry-run (preview only). Dry-run shows what would be converted without making changes." + } } } } } } } -} diff --git a/migration/README.md b/migration/README.md index 97a0c7b..ac91535 100644 --- a/migration/README.md +++ b/migration/README.md @@ -10,6 +10,72 @@ This module is designed to facilitate environment migrations (e.g. dev → prod - **Validate** migration files before importing - **Import** configurations to a target MC environment (with dry-run support) +## Quick Start: US → EU Workspace Migration + +### Prerequisites + +1. **Python 3.8+** with dependencies installed (`pip install -r requirements.txt`) +2. **Monte Carlo API credentials** configured in `configs/configs.ini`: + ```ini + [us_workspace] + mcd_id = your_us_key_id + mcd_token = your_us_token + + [eu_workspace] + mcd_id = your_eu_key_id + mcd_token = your_eu_token + ``` + +### Migration Workflow + +**Step 1: Export from US workspace** +```bash +python migration/workspace_migrator.py export --profile us_workspace +``` + +**Step 2: Map warehouses** (required for tags and monitors) + +Edit `migration/migration-data-exports/warehouse_mapping_template.json` to create `warehouse_mapping.json`: +```json +{ + "warehouse_mapping": { + "US Snowflake Warehouse": "EU Snowflake Warehouse", + "US BigQuery Project": "EU BigQuery Project" + } +} +``` + +**Step 3: Validate** (optional but recommended) +```bash +python migration/workspace_migrator.py validate --profile eu_workspace +``` + +**Step 4: Import with dry-run** (preview changes) +```bash +python migration/workspace_migrator.py import --profile eu_workspace \ + --entities tags,monitors --convert-to-ui +``` + +**Step 5: Import with force** (commit changes) +```bash +python migration/workspace_migrator.py import --profile eu_workspace \ + --entities tags,monitors --convert-to-ui --force yes +``` + +### Important Behaviors + +- **Warehouse mapping is required** for `tags` and `monitors` when warehouse names differ between workspaces +- **Audiences are skipped** if they already exist (not updated) +- **Monitors are code-managed** after import—use `--convert-to-ui` to make them UI-editable +- **Dry-run is default**—always preview before using `--force yes` +- **Import order**: Entities are imported in dependency order (blocklists → domains → tags → exclusion_windows → data_products → audiences → monitors) + +### Troubleshooting + +- **Logs**: Check `logs/workspace_migrator-YYYY-MM-DD.log` for detailed error messages +- **Unmapped warehouses**: Tags/monitors with unmapped warehouses are skipped with warnings +- **Validation errors**: Run `validate` command to check file format and required fields before importing +- **API errors**: Verify credentials in `configs/configs.ini` and check network connectivity ### Key Components diff --git a/requirements.txt b/requirements.txt index 88c6be8..cfa18e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -106,7 +106,7 @@ propcache==0.2.1 protobuf==5.29.1 psutil==6.1.0 py==1.11.0 -pycarlo==0.9.15 +pycarlo>=0.12.0 pycognito==2024.5.1 pycparser==2.22 pycryptodome==3.20.0