From b511cea9b3a05fb5f7a00fa565536ac2fe5b174a Mon Sep 17 00:00:00 2001 From: Nick Maludy Date: Tue, 18 Aug 2020 21:55:59 -0400 Subject: [PATCH] Remove Postgres backups --- CHANGES.md | 7 ++ README.md | 18 ++---- actions/full_backup.yaml | 3 - actions/postgres_backup.yaml | 23 ------- actions/workflows/full_backup.yaml | 40 ++---------- actions/workflows/postgres_backup.yaml | 89 -------------------------- pack.yaml | 4 +- 7 files changed, 20 insertions(+), 164 deletions(-) delete mode 100644 actions/postgres_backup.yaml delete mode 100644 actions/workflows/postgres_backup.yaml diff --git a/CHANGES.md b/CHANGES.md index cb85f32..833713c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,12 @@ # Change Log +## v1.1.0 + +* Removed the `backups.postgres_backup` action (Mistral is being removed from StackStorm). +* The `backups.full_backup` action now only backs up the MongoDB database. + + Contributed by Nick Maludy (@nmaludy Encore Technologies) + ## v1.0.5 * Forced release to included hotfix for `backups.full_backup` diff --git a/README.md b/README.md index cb7acbd..fa4a9d3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Backups Pack -This pack backs up the MongoDB and PostgreSQL databases in a StackStorm deployment. +This pack backs up the MongoDB database in a StackStorm deployment. It is currently best used on a single node deployment, but has the potential to be used for HA setups. @@ -10,7 +10,6 @@ The `st2actionrunner` node(s) that executes the actions must have the following packages installed: - `mongodb-org-tools` (for the `mongodump` tool) -- `postgresql` (for the `pg_dump` tool) # Configuration @@ -37,7 +36,7 @@ st2 key set mongodb.admin_username "admin" st2 key set -e mongodb.admin_password "Secret!" # set our mongodb host -st2 key set mongodb.mongodb_host "server01" +st2 key set mongodb.mongodb_host server01" ``` # Actions @@ -47,9 +46,8 @@ is built on: | Action | Description | |-------------------|-------------| -| `full_backup` | Performs a backup of both MongoDB and PostgreSQL databases | -| `mongodb_backup` | Performs a backup of the StackStorm MongoDB database | -| `postgres_backup` | Performs a backup of the StackStorm PostgreSQL database | +| `full_backup` | Performs a backup of the MongoDB database | +| `mongodb_backup` | Performs a backup of the StackStorm MongoDB database | ## Example @@ -59,9 +57,6 @@ st2 run backups.full_backup # perform backup of mongodb st2 run backups.mongodb_backup - -# perform backup of postgres -st2 run backups.postgres_backup ``` # Rules @@ -97,8 +92,5 @@ action: # TODO - Allow passwords and usernames to be set via config instead of being in the key/value store -- Allow PostgreSQL password to be passed in via parameters - Allow MongoDB username/password to be read in from the /etc/st2/st2.conf -- Implement backups of PostgreSQL database (allow all credentials/configs to be passed in with - parameters) -- Investigate using Orchestra or ActionChains instead of Mistral +- Backup "content" in /opt/stackstorm/packs, /opt/stackstorm/virtualenvs, configs, etc diff --git a/actions/full_backup.yaml b/actions/full_backup.yaml index cb7e157..803aea1 100644 --- a/actions/full_backup.yaml +++ b/actions/full_backup.yaml @@ -9,9 +9,6 @@ parameters: path: default: /opt/stackstorm/backups type: string - mistral_config: - default: /etc/mistral/mistral.conf - type: string mongodb_admin_username: default: "{{ st2kv.system.mongodb.admin_username }}" type: string diff --git a/actions/postgres_backup.yaml b/actions/postgres_backup.yaml deleted file mode 100644 index c2a1e6e..0000000 --- a/actions/postgres_backup.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -description: Performs a back of StackStorm Postgres database -enabled: true -runner_type: orquesta -entry_point: workflows/postgres_backup.yaml -name: postgres_backup -pack: backups -parameters: - path: - default: /opt/stackstorm/backups - type: string - date: - type: string - mistral_config: - default: /etc/mistral/mistral.conf - type: string - retention_days: - type: string - description: "Passed into the find command. Deletes files based on this value. To find files older than n days it should be +n . If you specify just n it will find files that are exactly n days old." - default: "{{ st2kv.system.backup.retention_days | string | default('+14', true) }}" - backup_timeout: - type: integer - default: 600 diff --git a/actions/workflows/full_backup.yaml b/actions/workflows/full_backup.yaml index fc587dc..b424ca6 100644 --- a/actions/workflows/full_backup.yaml +++ b/actions/workflows/full_backup.yaml @@ -1,9 +1,8 @@ --- version: '1.0' -description: Workflow that backs up the StackStorm MongoDB and Postgres databases +description: Workflow that backs up the StackStorm MongoDB database input: - path - - mistral_config - mongodb_admin_username - mongodb_admin_password - retention_days @@ -12,11 +11,9 @@ input: vars: - mongo_backup_success: false - - postgres_backup_success: false output: - mongodb_dump_archive: '{{ ctx().mongodb_dump_archive }}' - - postgres_dump_archive: '{{ ctx().postgres_dump_archive }}' tasks: date: @@ -29,7 +26,6 @@ tasks: - date: '{{ result().stdout }}' do: - mongodb_backup - - postgres_backup mongodb_backup: action: backups.mongodb_backup @@ -41,38 +37,16 @@ tasks: mongodb_host: '{{ ctx().mongodb_host }}' retention_days: '{{ ctx().retention_days }}' backup_timeout: '{{ ctx().backup_timeout }}' - next: - - when: "{{ completed() }}" + - when: "{{ succeeded() }}" publish: - - mongo_backup_success: "{{ true if succeeded() else false }}" + - mongo_backup_success: "{{ true }}" - mongodb_dump_archive: '{{ result().output.mongodb_dump_archive }}' - do: - - end - - postgres_backup: - action: backups.postgres_backup - input: - path: '{{ ctx().path }}' - date: '{{ ctx().date }}' - mistral_config: '{{ ctx().mistral_config }}' - retention_days: '{{ ctx().retention_days }}' - backup_timeout: '{{ ctx().backup_timeout }}' - next: - - when: "{{ completed() }}" - publish: - - postgres_backup_success: "{{ true if succeeded() else false }}" - - postgres_dump_archive: '{{ result().output.postgres_dump_archive }}' - do: - - end - - end: - action: core.noop - join: all - next: - - when: "{{ succeeded() and (ctx().mongo_backup_success and ctx().postgres_backup_success) }}" do: - noop - - when: "{{ succeeded() and (not ctx().mongo_backup_success or not ctx().postgres_backup_success) }}" + - when: "{{ failed() }}" + publish: + - mongo_backup_success: "{{ false }}" + - mongodb_dump_archive: '{{ result().output.mongodb_dump_archive }}' do: - fail diff --git a/actions/workflows/postgres_backup.yaml b/actions/workflows/postgres_backup.yaml deleted file mode 100644 index 5b51bdd..0000000 --- a/actions/workflows/postgres_backup.yaml +++ /dev/null @@ -1,89 +0,0 @@ ---- -version: '1.0' -description: Workflow that backs up the StackStorm Postgres database -input: - - path - - mistral_config - - date - - retention_days - - backup_timeout - -vars: - - postgres_dump_archive: "" - - run_success: true - -output: - - postgres_dump_archive: '{{ ctx().postgres_dump_archive }}' - -tasks: - main: - action: core.noop - next: - - when: '{{ succeeded() and (ctx().date) }}' - do: - - read_connection - - when: '{{ succeeded() and (not ctx().date) }}' - do: - - date - - date: - action: core.local - input: - cmd: "date +%Y%m%d_%H%M%S" - next: - - when: '{{ succeeded() }}' - publish: - - date: '{{ result().stdout }}' - do: - - read_connection - - read_connection: - action: core.local - input: - cmd: "grep -E '^connection' {{ ctx().mistral_config }}" - next: - - when: '{{ succeeded() }}' - publish: - - postgres_connection: "{{ result().stdout.split(' ')[2] }}" - - postgres_dump_archive: '{{ ctx().path }}/postgres_dump_{{ ctx().date }}.sql.gz' - do: - - mkdir_path - - mkdir_path: - action: core.local_sudo - input: - cmd: "mkdir -p '{{ ctx().path }}'" - next: - - when: '{{ succeeded() }}' - do: - - postgres_backup - - # Even though the action might timeout the back up eventually succeeds so we still want to - # remove the old backups to prevent the disk space from filling up. - postgres_backup: - action: core.local_sudo - input: - cmd: "pg_dump --compress=9 -f '{{ ctx().postgres_dump_archive }}' '{{ ctx().postgres_connection }}'" - timeout: "{{ ctx().backup_timeout }}" - next: - - when: '{{ succeeded() }}' - do: - - postgres_delete_old_files - - when: '{{ failed() }}' - publish: - - run_success: false - do: - - postgres_delete_old_files - - postgres_delete_old_files: - action: core.local_sudo - input: - cwd: '{{ ctx().path }}' - cmd: "find . -name 'postgres_dump_*' -mtime {{ ctx().retention_days }} | xargs --no-run-if-empty rm" - next: - - when: "{{ succeeded() and (ctx().run_success) }}" - do: - - noop - - when: "{{ succeeded() and (not ctx().run_success) }}" - do: - - fail diff --git a/pack.yaml b/pack.yaml index ff0eb1f..b6aa9a8 100644 --- a/pack.yaml +++ b/pack.yaml @@ -5,11 +5,9 @@ description: Pack to backup StackStorm databases keywords: - backup - database - - postgres - - postgresql - mongo - mongodb -version: 1.0.5 +version: 1.1.0 author: Encore Technologies email: code@encore.tech python_versions: