Skip to content

Palvarez/migration cli app #96

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pcalvarez wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Palvarez/migration cli app #96

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c9fc942
made minor changes to make migration module work through app
pcalvarez Feb 19, 2026
9477bd0
completed documentation and parser configuration
pcalvarez Mar 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions app/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ def __init__(self, **kwargs):
}
DEFAULT_MODE = "categories"
BINDINGS = [
Binding(
"q",
"quit",
"Quit",
tooltip="Quit the application",
),
Binding(
"r",
"app.switch_mode('readme')",
Expand Down
43 changes: 30 additions & 13 deletions app/executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from textual.binding import Binding
from textual.app import ComposeResult
from textual.containers import Grid, Center, Vertical, VerticalScroll, ScrollableContainer
from textual.widgets import Footer, Label, Markdown, Static, Input, Select, RichLog, TabbedContent
from textual.widgets import Footer, Label, Markdown, Static, Input, Select, RichLog, TabbedContent, SelectionList
from textual.screen import Screen, ModalScreen
from app.header import MCHeader
from textual import events, on, work
Expand Down Expand Up @@ -43,6 +43,11 @@ class ExecutorArgs(Vertical, can_focus=False, can_focus_children=True):
padding: 1;
text-align: right;
}
SelectionList {
height: 6;
min-height: 6;
max-height: 6;
}
}
}
"""
Expand Down Expand Up @@ -78,12 +83,15 @@ def compose(self) -> ComposeResult:
yield Label(f"{mandatory}{arg}")
if arg == 'profile':
yield Select((profile, profile) for profile in self.get_profiles())
elif arg == 'entities' and arguments[arg].get('choices'):
# Multi-select for entities
choices = arguments[arg]['choices']
yield SelectionList(*[(choice, choice, choice == 'all') for choice in choices], id='entities')
elif arguments[arg].get('choices'):
choices = arguments[arg]['choices']
yield Select((choice, choice) for choice in choices)
else:
if arguments[arg].get('choices'):
choices = arguments[arg]['choices']
yield Select((choice, choice) for choice in choices)
else:
yield Input(placeholder=arguments[arg]['help'])
yield Input(placeholder=arguments[arg]['help'])

def get_profiles(self):
"""Read headers from mcd profiles.ini"""
Expand Down Expand Up @@ -116,13 +124,16 @@ def update_grid_with_selected_subparser(self) -> None:
grid._add_child(Label(f"{mandatory}{arg}"))
if arg == 'profile':
grid._add_child(Select((profile, profile) for profile in self.get_profiles()))
elif arg == 'entities' and selected_subparser_arguments[arg].get('choices'):
# Multi-select for entities
choices = selected_subparser_arguments[arg]['choices']
grid._add_child(SelectionList(*[(choice, choice, choice == 'all') for choice in choices], id='entities'))
elif selected_subparser_arguments[arg].get('choices'):
choices = selected_subparser_arguments[arg]['choices']
grid._add_child(Select.from_values(choices))
else:
if selected_subparser_arguments[arg].get('choices'):
choices = selected_subparser_arguments[arg]['choices']
grid._add_child(Select.from_values(choices))
else:
grid._add_child(Input(placeholder=selected_subparser_arguments[arg]['help'],
tooltip=selected_subparser_arguments[arg]['help']))
grid._add_child(Input(placeholder=selected_subparser_arguments[arg]['help'],
tooltip=selected_subparser_arguments[arg]['help']))

nodes = grid.query_children().nodes
grid.mount(*nodes)
Expand Down Expand Up @@ -173,7 +184,13 @@ async def action_run_utility(self):
args.append(f"-{flag[0]}")
if '★' in element._Static__content:
mandatory = True
if isinstance(element, Input) or isinstance(element, Select):
if isinstance(element, SelectionList):
# Multi-select: join selected values with commas
selected = list(element.selected)
value = ','.join(selected) if selected else 'all'
args.append(value)
mandatory = False
elif isinstance(element, Input) or isinstance(element, Select):
value = element.value
if mandatory and value == '':
self.notify(f"'{flags[-1]}' is required", severity="error")
Expand Down
117 changes: 60 additions & 57 deletions lib/helpers/parser_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,10 +590,10 @@
},
"migration": {
"workspace_migrator.py": {
"description": "\nWorkspace migration utility for Monte Carlo configurations.\n\nExport, import, and validate domains, data products, and blocklists between environments.\n\nExamples:\n\t# Export all entities\n\tpython migration/workspace_migrator.py export --profile source_env\n\n\t# Export specific entities\n\tpython migration/workspace_migrator.py export --profile source_env --entities domains,blocklists\n\n\t# Import (dry-run by default)\n\tpython migration/workspace_migrator.py import --profile target_env\n\n\t# Import with force (commit changes)\n\tpython migration/workspace_migrator.py import --profile target_env --force yes\n\n\t# Validate migration files\n\tpython migration/workspace_migrator.py validate --profile target_env",
"description": "\nWorkspace migration utility for Monte Carlo configurations.\n\nExport, import, and validate all MC configurations between environments. Supports: blocklists, domains, tags, exclusion_windows, data_products, audiences, and monitors.\n\nExamples:\n\t# Export all entities\n\tpython migration/workspace_migrator.py export --profile source_env\n\n\t# Export specific entities\n\tpython migration/workspace_migrator.py export --profile source_env --entities domains,blocklists,tags\n\n\t# Import (dry-run by default)\n\tpython migration/workspace_migrator.py import --profile target_env\n\n\t# Import with force (commit changes)\n\tpython migration/workspace_migrator.py import --profile target_env --force yes\n\n\t# Validate migration files\n\tpython migration/workspace_migrator.py validate --profile target_env",
"subparsers": {
"export": {
"description": "Export domains, data products, and blocklists to CSV files.",
"description": "Export MC configurations to files. Exports blocklists, domains, tags, exclusion_windows, data_products, and audiences to CSV files. Exports monitors to YAML (MaC format).\n\nExample:\n\tpython migration/workspace_migrator.py export --profile source_env --entities all",
"help": "Export MC configurations to files.",
"arguments": {
"profile": {
Expand All @@ -604,16 +604,17 @@
"entities": {
"required": false,
"default": "all",
"help": "Comma-separated list of entities: blocklists,domains,tags,exclusion_windows,data_products,monitors (or 'all')"
"choices": ["all", "blocklists", "domains", "tags", "exclusion_windows", "data_products", "audiences", "monitors"],
"help": "Comma-separated list of entity types to export, or 'all' for all entities. Example: 'domains,blocklists,tags'. Entities are exported in dependency order."
},
"output_dir": {
"required": false,
"help": "Directory to write export files. Defaults to migration/migration-data-exports/"
"help": "Directory to write export files. Default: migration/migration-data-exports/"
}
}
},
"import": {
"description": "Import domains, data products, blocklists, and monitors from files. Dry-run by default.",
"description": "Import MC configurations from files. Imports blocklists, domains, tags, exclusion_windows, data_products, and audiences from CSV files. Imports monitors from YAML (MaC format). Dry-run by default.\n\nEntities are imported in dependency order: blocklists → domains → tags → exclusion_windows → data_products → audiences → monitors.\n\nWarehouse mapping is required for tags and monitors when warehouse names differ between source and target environments.\n\nFor monitors: Monitors imported via MaC are code-managed by default. Use --convert_to_ui flag to auto-convert after import, or use the 'convert-to-ui' subcommand separately after import.\n\nExample:\n\tpython migration/workspace_migrator.py import --profile target_env --entities tags,monitors --warehouse_map \"US Warehouse=EU Warehouse\" --convert_to_ui --force yes",
"help": "Import MC configurations from files.",
"arguments": {
"profile": {
Expand All @@ -624,70 +625,72 @@
"entities": {
"required": false,
"default": "all",
"help": "Comma-separated list of entities: blocklists,domains,tags,exclusion_windows,data_products,monitors (or 'all')"
"choices": ["all", "blocklists", "domains", "tags", "exclusion_windows", "data_products", "audiences", "monitors"],
"help": "Comma-separated list of entity types to import, or 'all' for all entities. Example: 'domains,blocklists,tags'. Entities are imported in dependency order."
},
"input_dir": {
"required": false,
"help": "Directory containing import files. Defaults to migration/migration-data-exports/"
"help": "Directory containing import files. Default: migration/migration-data-exports/"
},
"warehouse_map": {
"required": false,
"help": "Warehouse name mapping for tag migrations: 'Source1=Dest1,Source2=Dest2'. Or create warehouse_mapping.json in input_dir."
"help": "Warehouse name mapping for tags and monitors migrations. Required when warehouse names differ between source and target environments.\n\nFormat: 'SourceName1=DestName1,SourceName2=DestName2'\nExample: 'US Snowflake=EU Snowflake,US BigQuery=EU BigQuery'\n\nAlternatively, create warehouse_mapping.json in input_dir with format:\n{\n \"warehouse_mapping\": {\n \"Source Name\": \"Destination Name\"\n }\n}"
},
"force": {
"required": false,
"choices": ["yes", "no"],
"help": "Set to 'yes' to commit changes. Default is dry-run (preview only)."
},
"convert_to_ui": {
"required": false,
"action": "store_true",
"help": "Convert monitors to UI-editable after import."
"force": {
"required": false,
"choices": ["yes", "no"],
"help": "Set to 'yes' to commit changes. Default is dry-run (preview only). Dry-run shows what would be created/updated/skipped without making changes."
},
"convert_to_ui": {
"required": false,
"action": "store_true",
"help": "Convert monitors to UI-editable after import. Monitors imported via MaC are code-managed by default and cannot be edited in the UI. This flag converts them to UI-editable monitors."
}
}
}
},
"validate": {
"description": "Validate migration files before importing.",
"help": "Validate migration files.",
"arguments": {
"profile": {
"default": "default",
"required": false,
"help": "Specify an MCD profile name. Uses default otherwise."
},
"entities": {
"required": false,
"default": "all",
"help": "Comma-separated list of entities: blocklists,domains,tags,exclusion_windows,data_products,monitors (or 'all')"
},
"input_dir": {
"required": false,
"help": "Directory containing files to validate. Defaults to migration/migration-data-exports/"
},
"validate": {
"description": "Validate migration files before importing. Checks file format, required fields, and data integrity.\n\nExample:\n\tpython migration/workspace_migrator.py validate --profile target_env --entities all",
"help": "Validate migration files before importing.",
"arguments": {
"profile": {
"default": "default",
"required": false,
"help": "Specify an MCD profile name. Uses default otherwise."
},
"entities": {
"required": false,
"default": "all",
"choices": ["all", "blocklists", "domains", "tags", "exclusion_windows", "data_products", "audiences", "monitors"],
"help": "Comma-separated list of entity types to validate, or 'all' for all entities. Example: 'domains,blocklists,tags'"
},
"input_dir": {
"required": false,
"help": "Directory containing files to validate. Default: migration/migration-data-exports/"
}
}
}
},
"convert-to-ui": {
"description": "Convert monitors in a namespace from code-deployed to UI-editable.",
"help": "Convert monitors to UI-editable.",
"arguments": {
"profile": {
"default": "default",
"required": false,
"help": "Specify an MCD profile name. Uses default otherwise."
},
"namespace": {
"required": false,
"default": "migration",
"help": "Namespace containing monitors to convert. Defaults to 'migration'."
},
"force": {
"required": false,
"choices": ["yes", "no"],
"help": "Set to 'yes' to commit changes. Default is dry-run (preview only)."
},
"convert-to-ui": {
"description": "Convert monitors in a namespace from code-deployed to UI-editable. Monitors imported via MaC (Monitors as Code) are code-managed by default and cannot be edited in the Monte Carlo UI. This command converts them to UI-editable monitors.\n\nUse this subcommand if you imported monitors without the --convert_to_ui flag, or if you want to convert monitors from a previous import.\n\nAfter conversion, monitors move from the custom namespace (e.g., 'migration') to the 'ui' namespace.\n\nExample:\n\tpython migration/workspace_migrator.py convert-to-ui --profile target_env --namespace migration --force yes",
"help": "Convert monitors to UI-editable.",
"arguments": {
"profile": {
"default": "default",
"required": false,
"help": "Specify an MCD profile name. Uses default otherwise."
},
"namespace": {
"required": false,
"default": "migration",
"help": "Namespace containing monitors to convert. Defaults to 'migration'. Monitors imported via the migration tool are placed in this namespace by default."
},
"force": {
"required": false,
"choices": ["yes", "no"],
"help": "Set to 'yes' to commit changes. Default is dry-run (preview only). Dry-run shows what would be converted without making changes."
}
}
}
}
}
}
}
}
66 changes: 66 additions & 0 deletions migration/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,72 @@ This module is designed to facilitate environment migrations (e.g. dev → prod
- **Validate** migration files before importing
- **Import** configurations to a target MC environment (with dry-run support)

## Quick Start: US → EU Workspace Migration

### Prerequisites

1. **Python 3.8+** with dependencies installed (`pip install -r requirements.txt`)
2. **Monte Carlo API credentials** configured in `configs/configs.ini`:
```ini
[us_workspace]
mcd_id = your_us_key_id
mcd_token = your_us_token

[eu_workspace]
mcd_id = your_eu_key_id
mcd_token = your_eu_token
```

### Migration Workflow

**Step 1: Export from US workspace**
```bash
python migration/workspace_migrator.py export --profile us_workspace
```

**Step 2: Map warehouses** (required for tags and monitors)

Edit `migration/migration-data-exports/warehouse_mapping_template.json` to create `warehouse_mapping.json`:
```json
{
"warehouse_mapping": {
"US Snowflake Warehouse": "EU Snowflake Warehouse",
"US BigQuery Project": "EU BigQuery Project"
}
}
```

**Step 3: Validate** (optional but recommended)
```bash
python migration/workspace_migrator.py validate --profile eu_workspace
```

**Step 4: Import with dry-run** (preview changes)
```bash
python migration/workspace_migrator.py import --profile eu_workspace \
--entities tags,monitors --convert-to-ui
```

**Step 5: Import with force** (commit changes)
```bash
python migration/workspace_migrator.py import --profile eu_workspace \
--entities tags,monitors --convert-to-ui --force yes
```

### Important Behaviors

- **Warehouse mapping is required** for `tags` and `monitors` when warehouse names differ between workspaces
- **Audiences are skipped** if they already exist (not updated)
- **Monitors are code-managed** after import—use `--convert-to-ui` to make them UI-editable
- **Dry-run is default**—always preview before using `--force yes`
- **Import order**: Entities are imported in dependency order (blocklists → domains → tags → exclusion_windows → data_products → audiences → monitors)

### Troubleshooting

- **Logs**: Check `logs/workspace_migrator-YYYY-MM-DD.log` for detailed error messages
- **Unmapped warehouses**: Tags/monitors with unmapped warehouses are skipped with warnings
- **Validation errors**: Run `validate` command to check file format and required fields before importing
- **API errors**: Verify credentials in `configs/configs.ini` and check network connectivity

### Key Components

Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ propcache==0.2.1
protobuf==5.29.1
psutil==6.1.0
py==1.11.0
pycarlo==0.9.15
pycarlo>=0.12.0
pycognito==2024.5.1
pycparser==2.22
pycryptodome==3.20.0
Expand Down