diff --git a/cfn-resources/README.md b/cfn-resources/README.md index a55bb7a55..098c4dbd0 100644 --- a/cfn-resources/README.md +++ b/cfn-resources/README.md @@ -9,56 +9,54 @@ Note these are also hosted on AWS CloudFormation Public Registry under Third Par ### Resource Status Table -| # | Resource | Status | Examples | Local Testing Scripts | -|-----|-------------------------------------------------------------|-------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| -| 1 | alert-configuration | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/alert-configuration/alert-configuration.json) | [./alert-configuration/test](./alert-configuration/test) | -| 2 | auditing | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/auditing/auditing.json) | [./auditing/test](./auditing/test) | -| 3 | cloud-backup-restore-jobs | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cloud-backup-restore-jobs/restore.json) | [./cloud-backup-restore-jobs/test](./cloud-backup-restore-jobs/test) | -| 4 | cloud-backup-schedule | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cloud-backup-schedule/cloudBackupSchedule.json) | [./cloud-backup-schedule/test](./cloud-backup-schedule/test) | -| 5 | cloud-backup-snapshot | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cloud-backup-snapshot/snapshot.json) | [./cloud-backup-snapshot/test](./cloud-backup-snapshot/test) | -| 6 | cloud-backup-snapshot-export-bucket | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cloud-backup-snapshot-export-bucket/CloudBackupSnapshotExportBucket.json) | [./cloud-backup-snapshot-export-bucket/test](./cloud-backup-snapshot-export-bucket/test) | -| 7 | cluster | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cluster/cluster.json) | [./cluster/test](./cluster/test) | -| 8 | custom-dns-configuration-cluster-aws | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/custom-dns-configuration-cluster-aws/CustomDnsConfigurationClusterAws.json) | [./custom-db-role/test](./custom-db-role/test) | -| 9 | custom-db-role | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/custom-db-role/custom-db-role.json) | [./custom-dns-configuration-cluster-aws/test](./custom-dns-configuration-cluster-aws/test) | -| 10 | database-user | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/database-user/user.json) | [./database-user/test](./database-user/test) | -| 11 | datalakes | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/data-lakes/datalake.json) | [./datalakes/test](./datalakes/test) | -| 12 | encryption-at-rest | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/encryption-at-rest/encryption-at-rest.json) | [./encryption-at-rest/test](./encryption-at-rest/test) | -| 13 | federated-settings-org-role-mapping | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/federated-settings-org-role-mapping/federatedSettingsOrgRoleMapping.json) | [./federated-settings-org-role-mapping/test](./federated-settings-org-role-mapping/test) | -| 14 | global-cluster-config | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/global-cluster-config/global-cluster-config.json) | [./global-cluster-config/test](./global-cluster-config/test) | -| 15 | ldap-configuration | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/ldap-configuration/LDAPConfiguration.json) | [./ldap-configuration/test](./ldap-configuration/test) | -| 16 | ldap-verify | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/LDAPVerify/LDAPVerify.json) | [./ldap-verify/test](./ldap-verify/test) | -| 17 | maintenance-window | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/maintenance-window/maintenance-window.json) | [./maintenance-window/test](./maintenance-window/test) | -| 18 | network-container | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/network-container/container.json) | [./network-container/test](./network-container/test) | -| 19 | network-peering | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/network-peering/peering.json) | [./network-peering/test](./network-peering/test) | -| 20 | online-archive | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/online-archive/online-archive.json) | [./online-archive/test](./online-archive/test) | -| 21 | org-invitation | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/org-invitation/org-invitation.json) | [./org-invitation/test](./org-invitation/test) | -| 22 | private-endpoint | ![Build](https://img.shields.io/badge/Deprecated-red) | [example](../examples/private-endpoint/privateEndpoint.json) | [./private-endpoint/test](./private-endpoint/test) | -| 23 | private-endpoint-adl | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/private-endpoint-adl/endpoint-adl.json) | [./private-endpoint-adl/test](./private-endpoint-adl/test) | -| 24 | private-endpoint-regional-mode | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/private-endpoint-regional-mode/privateEndpointRegionalMode.json) | [./private-endpoint-regional-mode/test](./private-endpoint-regional-mode/test) | -| 25 | project | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/project/project.json) | [./project/test](./project/test) | -| 26 | project-invitation | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/project-invitation/project-invitation.json) | [./project-invitation/test](./project-invitation/test) | -| 27 | project-ip-access-list | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/project-ip-access-list/ip-access-list.yaml) | [./project-ip-access-list/test](./project-ip-access-list/test) | -| 28 | search-index | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/search-index/searchIndex.json) | [./search-indexes/test](./search-indexes/test) | -| 29 | serverless-instance | ![Build](https://img.shields.io/badge/Deprecated-red) | [example](../examples/serverless-instance/serverless-instance.json) | [./serverless-instance/test](./serverless-instance/test) | -| 30 | teams | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/teams/teams.json) | [./teams/test](./teams/test) | -| 31 | third-party-integration | ![Build](https://img.shields.io/badge/GA-green) | [example files](../examples/thirdpartyintegrations) | [./third-party-integration/test](./third-party-integration/test) | -| 32 | trigger | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/trigger/trigger.json) | [./trigger/test](./trigger/test) | -| 33 | X509AuthenticationDatabaseUser | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/x509-authentication-db-user/x509-authentication-db-user.json) | [./x509-authentication-database-user/test](./x509-authentication-database-user/test) | -| 34 | federated-database-instance | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/federated-database-instance/federatedDatabaseInstance.json) | [./federated-database-instance/test](./federated-database-instance/test) | -| 35 | privatelink-endpoint-service-data-federation-online-archive | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/privatelink-endpoint-service-data-federation-online-archive/privatelink-endpoint-service-data-federation-online-archive.json) | [./privatelink-endpoint-service-data-federation-online-archive/test](./privatelink-endpoint-service-data-federation-online-archive/test) | -| 36 | federated-query-limit | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/federated-query-limit/federatedQueryLimit.json) | [./federated-query-limit/test](./federated-query-limit/test) | -| 37 | serverless-private-endpoint | ![Build](https://img.shields.io/badge/Deprecated-red) | [example](../examples/serverless-private-endpoint/serverless-private-endpoint-with-aws-private-endpoint.json) | [./serverless-private-endpoint/test](./serverless-private-endpoint/test) | -| 38 | api-key | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/api-key/api-key.json) | [./api-key/test](./api-key/test) | -| 39 | access-list-api-key | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/access-list-api-key/access-list-api-key.json) | [./access-list-api-key/test](./access-list-api-key/test) | -| 40 | data-lake-pipeline | ![Build](https://img.shields.io/badge/Deprecated-red) | [example](../examples/data-lake-pipeline/datalakepipeline.json) | [./data-lake-pipeline/test](./data-lake-pipeline/test) | -| 41 | organization | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/organization/organization.json) | [./organization/test](./organization/test) | -| 42 | cloud-outage-simulation | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cluster-outage-simulation/cluster-outage-simulation.json) | [./cloud-outage-simulation/test](./cl-outage-simulation/test) | -| 43 | private-endpoint-service | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/private-endpoint/privateEndpointV2.json) | [./private-endpoint-service/test](./private-endpoint-service/test) | -| 44 | private-endpoint-aws | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/private-endpoint/privateEndpointV2.json) | [./private-endpoint-aws/test](./private-endpoint-aws/test) -| 45 | search-deployment | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/search-deployment/search-deployment.json) | [./search-deployment/test](./search-deployment/test) -| 46 | stream-instance | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/atlas-streams/stream-instance/stream-instance.json) | [./stream-instance/test](./stream-instance/test) -| 47 | stream-connection | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/atlas-streams/stream-connection/stream-connection.json) | [./stream-connection/test](./stream-connection/test) -|47 | resource-policy | ![Build](https://img.shields.io/badge/Beta-yellow) | [example](../examples/resource-policy/resource-policy.json) | [./resource-policy/test](./resource-policy/test) +| Resource | Status | Examples | Local Testing Scripts | +|-------------------------------------------------------------|-------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| +| alert-configuration | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/alert-configuration/alert-configuration.json) | [./alert-configuration/test](./alert-configuration/test) | +| auditing | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/auditing/auditing.json) | [./auditing/test](./auditing/test) | +| cloud-backup-restore-jobs | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cloud-backup-restore-jobs/restore.json) | [./cloud-backup-restore-jobs/test](./cloud-backup-restore-jobs/test) | +| cloud-backup-schedule | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cloud-backup-schedule/cloudBackupSchedule.json) | [./cloud-backup-schedule/test](./cloud-backup-schedule/test) | +| cloud-backup-snapshot | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cloud-backup-snapshot/snapshot.json) | [./cloud-backup-snapshot/test](./cloud-backup-snapshot/test) | +| cloud-backup-snapshot-export-bucket | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cloud-backup-snapshot-export-bucket/CloudBackupSnapshotExportBucket.json) | [./cloud-backup-snapshot-export-bucket/test](./cloud-backup-snapshot-export-bucket/test) | +| cluster | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cluster/cluster.json) | [./cluster/test](./cluster/test) | +| custom-dns-configuration-cluster-aws | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/custom-dns-configuration-cluster-aws/CustomDnsConfigurationClusterAws.json) | [./custom-db-role/test](./custom-db-role/test) | +| custom-db-role | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/custom-db-role/custom-db-role.json) | [./custom-dns-configuration-cluster-aws/test](./custom-dns-configuration-cluster-aws/test) | +| database-user | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/database-user/user.json) | [./database-user/test](./database-user/test) | +| encryption-at-rest | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/encryption-at-rest/encryption-at-rest.json) | [./encryption-at-rest/test](./encryption-at-rest/test) | +| federated-settings-org-role-mapping | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/federated-settings-org-role-mapping/federatedSettingsOrgRoleMapping.json) | [./federated-settings-org-role-mapping/test](./federated-settings-org-role-mapping/test) | +| global-cluster-config | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/global-cluster-config/global-cluster-config.json) | [./global-cluster-config/test](./global-cluster-config/test) | +| ldap-configuration | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/ldap-configuration/LDAPConfiguration.json) | [./ldap-configuration/test](./ldap-configuration/test) | +| ldap-verify | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/LDAPVerify/LDAPVerify.json) | [./ldap-verify/test](./ldap-verify/test) | +| maintenance-window | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/maintenance-window/maintenance-window.json) | [./maintenance-window/test](./maintenance-window/test) | +| network-container | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/network-container/container.json) | [./network-container/test](./network-container/test) | +| network-peering | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/network-peering/peering.json) | [./network-peering/test](./network-peering/test) | +| online-archive | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/online-archive/online-archive.json) | [./online-archive/test](./online-archive/test) | +| org-invitation | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/org-invitation/org-invitation.json) | [./org-invitation/test](./org-invitation/test) | +| private-endpoint | ![Build](https://img.shields.io/badge/Deprecated-red) | [example](../examples/private-endpoint/privateEndpoint.json) | [./private-endpoint/test](./private-endpoint/test) | +| private-endpoint-adl | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/private-endpoint-adl/endpoint-adl.json) | [./private-endpoint-adl/test](./private-endpoint-adl/test) | +| private-endpoint-regional-mode | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/private-endpoint-regional-mode/privateEndpointRegionalMode.json) | [./private-endpoint-regional-mode/test](./private-endpoint-regional-mode/test) | +| project | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/project/project.json) | [./project/test](./project/test) | +| project-invitation | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/project-invitation/project-invitation.json) | [./project-invitation/test](./project-invitation/test) | +| project-ip-access-list | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/project-ip-access-list/ip-access-list.yaml) | [./project-ip-access-list/test](./project-ip-access-list/test) | +| search-index | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/search-index/searchIndex.json) | [./search-indexes/test](./search-indexes/test) | +| serverless-instance | ![Build](https://img.shields.io/badge/Deprecated-red) | [example](../examples/serverless-instance/serverless-instance.json) | [./serverless-instance/test](./serverless-instance/test) | +| teams | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/teams/teams.json) | [./teams/test](./teams/test) | +| third-party-integration | ![Build](https://img.shields.io/badge/GA-green) | [example files](../examples/thirdpartyintegrations) | [./third-party-integration/test](./third-party-integration/test) | +| trigger | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/trigger/trigger.json) | [./trigger/test](./trigger/test) | +| X509AuthenticationDatabaseUser | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/x509-authentication-db-user/x509-authentication-db-user.json) | [./x509-authentication-database-user/test](./x509-authentication-database-user/test) | +| federated-database-instance | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/federated-database-instance/federatedDatabaseInstance.json) | [./federated-database-instance/test](./federated-database-instance/test) | +| privatelink-endpoint-service-data-federation-online-archive | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/privatelink-endpoint-service-data-federation-online-archive/privatelink-endpoint-service-data-federation-online-archive.json) | [./privatelink-endpoint-service-data-federation-online-archive/test](./privatelink-endpoint-service-data-federation-online-archive/test) | +| federated-query-limit | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/federated-query-limit/federatedQueryLimit.json) | [./federated-query-limit/test](./federated-query-limit/test) | +| serverless-private-endpoint | ![Build](https://img.shields.io/badge/Deprecated-red) | [example](../examples/serverless-private-endpoint/serverless-private-endpoint-with-aws-private-endpoint.json) | [./serverless-private-endpoint/test](./serverless-private-endpoint/test) | +| api-key | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/api-key/api-key.json) | [./api-key/test](./api-key/test) | +| access-list-api-key | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/access-list-api-key/access-list-api-key.json) | [./access-list-api-key/test](./access-list-api-key/test) | +| organization | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/organization/organization.json) | [./organization/test](./organization/test) | +| cloud-outage-simulation | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/cluster-outage-simulation/cluster-outage-simulation.json) | [./cloud-outage-simulation/test](./cl-outage-simulation/test) | +| private-endpoint-service | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/private-endpoint/privateEndpointV2.json) | [./private-endpoint-service/test](./private-endpoint-service/test) | +| private-endpoint-aws | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/private-endpoint/privateEndpointV2.json) | [./private-endpoint-aws/test](./private-endpoint-aws/test) | +| search-deployment | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/search-deployment/search-deployment.json) | [./search-deployment/test](./search-deployment/test) | +| stream-instance | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/atlas-streams/stream-instance/stream-instance.json) | [./stream-instance/test](./stream-instance/test) | +| stream-connection | ![Build](https://img.shields.io/badge/GA-green) | [example](../examples/atlas-streams/stream-connection/stream-connection.json) | [./stream-connection/test](./stream-connection/test) | +| resource-policy | ![Build](https://img.shields.io/badge/Beta-yellow) | [example](../examples/resource-policy/resource-policy.json) | [./resource-policy/test](./resource-policy/test) | ## Resource Import Operations @@ -128,7 +126,3 @@ Each resource has a folder called `test` with 3 items: 3. **.sample-template.yaml Sample real cloudformation template you can run with `aws cloudformation create-stack` or using [../../quickstart-mongodb-atlas/scripts/launch-x-quickstart.sh]( ../../quickstart-mongodb-atlas/scripts/launch-x-quickstart.sh) _See each resource README for specific testing documentation._ - - - - diff --git a/cfn-resources/data-lake-pipeline/.rpdk-config b/cfn-resources/data-lake-pipeline/.rpdk-config deleted file mode 100644 index 08601a3d9..000000000 --- a/cfn-resources/data-lake-pipeline/.rpdk-config +++ /dev/null @@ -1,22 +0,0 @@ -{ - "artifact_type": "RESOURCE", - "typeName": "MongoDB::Atlas::DataLakePipeline", - "language": "go", - "runtime": "provided.al2", - "entrypoint": "bootstrap", - "testEntrypoint": "bootstrap", - "settings": { - "version": false, - "subparser_name": null, - "verbose": 0, - "force": false, - "type_name": null, - "artifact_type": null, - "endpoint_url": null, - "region": null, - "target_schemas": [], - "import_path": "github.com/mongodb/mongodbatlas-cloudformation-resources/data-lake-pipeline", - "protocolVersion": "2.0.0", - "pluginVersion": "2.0.4" - } -} diff --git a/cfn-resources/data-lake-pipeline/Makefile b/cfn-resources/data-lake-pipeline/Makefile deleted file mode 100644 index 296ff567e..000000000 --- a/cfn-resources/data-lake-pipeline/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -.PHONY: build test clean -tags=logging callback metrics scheduler -cgo=0 -goos=linux -goarch=amd64 -CFNREP_GIT_SHA?=$(shell git rev-parse HEAD) -ldXflags=-s -w -X github.com/mongodb/mongodbatlas-cloudformation-resources/util.defaultLogLevel=info -X github.com/mongodb/mongodbatlas-cloudformation-resources/version.Version=${CFNREP_GIT_SHA} -ldXflagsD=-X github.com/mongodb/mongodbatlas-cloudformation-resources/util.defaultLogLevel=debug -X github.com/mongodb/mongodbatlas-cloudformation-resources/version.Version=${CFNREP_GIT_SHA} - -build: - cfn generate - env GOOS=$(goos) CGO_ENABLED=$(cgo) GOARCH=$(goarch) go build -ldflags="$(ldXflags)" -tags="$(tags)" -o bin/bootstrap cmd/main.go - -debug: - cfn generate - env GOOS=$(goos) CGO_ENABLED=$(cgo) GOARCH=$(goarch) go build -ldflags="$(ldXflagsD)" -tags="$(tags)" -o bin/bootstrap cmd/main.go - -clean: - rm -rf bin - -create-test-resources: - @echo "==> Creating test files for contract testing" - ./test/contract-testing/cfn-test-create-inputs.sh - -delete-test-resources: - @echo "==> Delete test resources used for contract testing" - ./test/cfn-test-delete-inputs.sh - -run-contract-testing: - @echo "==> Run contract testing" - make build - sam local start-lambda & - cfn test --function-name TestEntrypoint --verbose diff --git a/cfn-resources/data-lake-pipeline/README.md b/cfn-resources/data-lake-pipeline/README.md deleted file mode 100644 index c72568579..000000000 --- a/cfn-resources/data-lake-pipeline/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# MongoDB::Atlas::DataLakePipeline - -**WARNING:** Data Lake is deprecated. As of September 2024, Data Lake is deprecated. If you use Data Lake, you should migrate to alternative solutions before the service is removed. To learn more, see - -## Description -Resource for managing [DataLakePipelines](https://www.mongodb.com/docs/api/doc/atlas-admin-api-v2/operation/operation-createpipeline) -## Requirements - -Set up an AWS profile to securely give CloudFormation access to your Atlas credentials. -For instructions on setting up a profile, [see here](/README.md#mongodb-atlas-api-keys-credential-management). - -## Attributes and Parameters - -See the [resource docs](docs/README.md). - -## Cloudformation Examples - -See the examples [CFN Template](/examples/data-lake-pipeline/datalakepipeline.json) for example resource. diff --git a/cfn-resources/data-lake-pipeline/cmd/main.go b/cfn-resources/data-lake-pipeline/cmd/main.go deleted file mode 100644 index b98a32d6e..000000000 --- a/cfn-resources/data-lake-pipeline/cmd/main.go +++ /dev/null @@ -1,85 +0,0 @@ -// Code generated by 'cfn generate', changes will be undone by the next invocation. DO NOT EDIT. -package main - -import ( - "errors" - "fmt" - "log" - - "github.com/aws-cloudformation/cloudformation-cli-go-plugin/cfn" - "github.com/aws-cloudformation/cloudformation-cli-go-plugin/cfn/handler" - "github.com/mongodb/mongodbatlas-cloudformation-resources/data-lake-pipeline/cmd/resource" -) - -// Handler is a container for the CRUDL actions exported by resources -type Handler struct{} - -// Create wraps the related Create function exposed by the resource code -func (r *Handler) Create(req handler.Request) handler.ProgressEvent { - return wrap(req, resource.Create) -} - -// Read wraps the related Read function exposed by the resource code -func (r *Handler) Read(req handler.Request) handler.ProgressEvent { - return wrap(req, resource.Read) -} - -// Update wraps the related Update function exposed by the resource code -func (r *Handler) Update(req handler.Request) handler.ProgressEvent { - return wrap(req, resource.Update) -} - -// Delete wraps the related Delete function exposed by the resource code -func (r *Handler) Delete(req handler.Request) handler.ProgressEvent { - return wrap(req, resource.Delete) -} - -// List wraps the related List function exposed by the resource code -func (r *Handler) List(req handler.Request) handler.ProgressEvent { - return wrap(req, resource.List) -} - -// main is the entry point of the application. -func main() { - cfn.Start(&Handler{}) -} - -type handlerFunc func(handler.Request, *resource.Model, *resource.Model) (handler.ProgressEvent, error) - -func wrap(req handler.Request, f handlerFunc) (response handler.ProgressEvent) { - defer func() { - // Catch any panics and return a failed ProgressEvent - if r := recover(); r != nil { - err, ok := r.(error) - if !ok { - err = errors.New(fmt.Sprint(r)) - } - - log.Printf("Trapped error in handler: %v", err) - - response = handler.NewFailedEvent(err) - } - }() - - // Populate the previous model - prevModel := &resource.Model{} - if err := req.UnmarshalPrevious(prevModel); err != nil { - log.Printf("Error unmarshaling prev model: %v", err) - return handler.NewFailedEvent(err) - } - - // Populate the current model - currentModel := &resource.Model{} - if err := req.Unmarshal(currentModel); err != nil { - log.Printf("Error unmarshaling model: %v", err) - return handler.NewFailedEvent(err) - } - - response, err := f(req, prevModel, currentModel) - if err != nil { - log.Printf("Error returned from handler function: %v", err) - return handler.NewFailedEvent(err) - } - - return response -} diff --git a/cfn-resources/data-lake-pipeline/cmd/resource/config.go b/cfn-resources/data-lake-pipeline/cmd/resource/config.go deleted file mode 100644 index 4d9eb7831..000000000 --- a/cfn-resources/data-lake-pipeline/cmd/resource/config.go +++ /dev/null @@ -1,19 +0,0 @@ -// Code generated by 'cfn generate', changes will be undone by the next invocation. DO NOT EDIT. -// Updates to this type are made my editing the schema file and executing the 'generate' command. -package resource - -import "github.com/aws-cloudformation/cloudformation-cli-go-plugin/cfn/handler" - -// TypeConfiguration is autogenerated from the json schema -type TypeConfiguration struct { -} - -// Configuration returns a resource's configuration. -func Configuration(req handler.Request) (*TypeConfiguration, error) { - // Populate the type configuration - typeConfig := &TypeConfiguration{} - if err := req.UnmarshalTypeConfig(typeConfig); err != nil { - return typeConfig, err - } - return typeConfig, nil -} diff --git a/cfn-resources/data-lake-pipeline/cmd/resource/model.go b/cfn-resources/data-lake-pipeline/cmd/resource/model.go deleted file mode 100644 index b5c6658ff..000000000 --- a/cfn-resources/data-lake-pipeline/cmd/resource/model.go +++ /dev/null @@ -1,46 +0,0 @@ -// Code generated by 'cfn generate', changes will be undone by the next invocation. DO NOT EDIT. -// Updates to this type are made my editing the schema file and executing the 'generate' command. -package resource - -// Model is autogenerated from the json schema -type Model struct { - Id *string `json:",omitempty"` - Profile *string `json:",omitempty"` - ProjectId *string `json:",omitempty"` - State *string `json:",omitempty"` - Name *string `json:",omitempty"` - CreatedDate *string `json:",omitempty"` - LastUpdatedDate *string `json:",omitempty"` - Sink *Sink `json:",omitempty"` - Source *Source `json:",omitempty"` - Transformations []Transformations `json:",omitempty"` -} - -// Sink is autogenerated from the json schema -type Sink struct { - Type *string `json:",omitempty"` - MetadataProvider *string `json:",omitempty"` - MetadataRegion *string `json:",omitempty"` - PartitionFields []PartitionFields `json:",omitempty"` -} - -// PartitionFields is autogenerated from the json schema -type PartitionFields struct { - FieldName *string `json:",omitempty"` - Order *int `json:",omitempty"` -} - -// Source is autogenerated from the json schema -type Source struct { - Type *string `json:",omitempty"` - ClusterName *string `json:",omitempty"` - CollectionName *string `json:",omitempty"` - DatabaseName *string `json:",omitempty"` - GroupId *string `json:",omitempty"` -} - -// Transformations is autogenerated from the json schema -type Transformations struct { - Field *string `json:",omitempty"` - Type *string `json:",omitempty"` -} diff --git a/cfn-resources/data-lake-pipeline/cmd/resource/resource.go b/cfn-resources/data-lake-pipeline/cmd/resource/resource.go deleted file mode 100644 index 99bd6a7f3..000000000 --- a/cfn-resources/data-lake-pipeline/cmd/resource/resource.go +++ /dev/null @@ -1,348 +0,0 @@ -// Copyright 2023 MongoDB Inc -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package resource - -import ( - ctx "context" - "fmt" - "net/http" - "time" - - "github.com/aws-cloudformation/cloudformation-cli-go-plugin/cfn/handler" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/cloudformation" - "github.com/mongodb/mongodbatlas-cloudformation-resources/profile" - "github.com/mongodb/mongodbatlas-cloudformation-resources/util" - "github.com/mongodb/mongodbatlas-cloudformation-resources/util/constants" - progress_events "github.com/mongodb/mongodbatlas-cloudformation-resources/util/progressevent" - "github.com/mongodb/mongodbatlas-cloudformation-resources/util/validator" - admin20231115014 "go.mongodb.org/atlas-sdk/v20231115014/admin" -) - -var CreateRequiredFields = []string{constants.ProjectID, constants.Name, constants.Sink} -var ReadRequiredFields = []string{constants.ProjectID, constants.Name} -var UpdateRequiredFields = []string{constants.ProjectID, constants.Name, constants.Sink} -var DeleteRequiredFields = []string{constants.ProjectID, constants.Name} -var ListRequiredFields = []string{constants.ProjectID} - -const ( - AlreadyExists = "already exists" -) - -func setup() { - util.SetupLogger("mongodb-atlas-federated-query-limit") -} - -func Create(req handler.Request, prevModel *Model, currentModel *Model) (handler.ProgressEvent, error) { - setup() - modelValidation := validator.ValidateModel(CreateRequiredFields, currentModel) - if modelValidation != nil { - return *modelValidation, nil - } - - if currentModel.Profile == nil || *currentModel.Profile == "" { - currentModel.Profile = aws.String(profile.DefaultProfile) - } - - client, peErr := util.NewAtlasClient(&req, currentModel.Profile) - if peErr != nil { - return *peErr, nil - } - - groupID := *currentModel.ProjectId - dataLakeIntegrationPipeline := generateDataLakeIntegrationPipeline(currentModel) - - pe, response, err := client.Atlas20231115014.DataLakePipelinesApi.CreatePipeline(ctx.Background(), groupID, dataLakeIntegrationPipeline).Execute() - - if err != nil { - if response.StatusCode == http.StatusBadRequest { - return handler.ProgressEvent{ - OperationStatus: handler.Failed, - Message: err.Error(), - HandlerErrorCode: cloudformation.HandlerErrorCodeAlreadyExists}, nil - } - return handleError(response, err) - } - - model := ReadResponseModelGeneration(pe) - if model == nil { - errorMsg := "Response model from the API is empty or nil " - return progress_events.GetFailedEventByResponse(errorMsg, response), nil - } - model.Profile = currentModel.Profile - return handler.ProgressEvent{ - OperationStatus: handler.Success, - Message: "Create Completed", - ResourceModel: model}, nil -} - -func generateDataLakeIntegrationPipeline(currentModel *Model) *admin20231115014.DataLakeIngestionPipeline { - partitionFieldsArr := make([]admin20231115014.DataLakePipelinesPartitionField, len(currentModel.Sink.PartitionFields)) - for i, partitionField := range currentModel.Sink.PartitionFields { - partitionFieldsArr[i] = admin20231115014.DataLakePipelinesPartitionField{ - FieldName: util.SafeString(partitionField.FieldName), - Order: util.SafeInt(partitionField.Order), - } - } - - transformationsFieldsArr := make([]admin20231115014.FieldTransformation, len(currentModel.Transformations)) - for i, transformation := range currentModel.Transformations { - transformationsFieldsArr[i] = admin20231115014.FieldTransformation{ - Field: transformation.Field, - Type: transformation.Type, - } - } - - dataLakeIntegrationPipeline := admin20231115014.DataLakeIngestionPipeline{ - GroupId: currentModel.ProjectId, - Name: currentModel.Name, - Sink: &admin20231115014.IngestionSink{ - MetadataProvider: currentModel.Sink.MetadataProvider, - MetadataRegion: currentModel.Sink.MetadataRegion, - PartitionFields: &partitionFieldsArr, - }, - Source: &admin20231115014.IngestionSource{ - Type: currentModel.Source.Type, - ClusterName: currentModel.Source.ClusterName, - CollectionName: currentModel.Source.CollectionName, - DatabaseName: currentModel.Source.DatabaseName, - GroupId: currentModel.Source.GroupId, - }, - Transformations: &transformationsFieldsArr, - } - - return &dataLakeIntegrationPipeline -} - -func Read(req handler.Request, prevModel *Model, currentModel *Model) (handler.ProgressEvent, error) { - setup() - - modelValidation := validator.ValidateModel(ReadRequiredFields, currentModel) - if modelValidation != nil { - return *modelValidation, nil - } - - if currentModel.Profile == nil || *currentModel.Profile == "" { - currentModel.Profile = aws.String(profile.DefaultProfile) - } - - client, peErr := util.NewAtlasClient(&req, currentModel.Profile) - if peErr != nil { - return *peErr, nil - } - - groupID := *currentModel.ProjectId - pipelineName := *currentModel.Name - - pe, response, err := client.Atlas20231115014.DataLakePipelinesApi.GetPipeline(ctx.Background(), groupID, pipelineName).Execute() - - if err != nil { - return handleError(response, err) - } - - model := ReadResponseModelGeneration(pe) - if model == nil { - errorMsg := "Response model from the API is empty or nil " - return progress_events.GetFailedEventByResponse(errorMsg, response), nil - } - model.Profile = currentModel.Profile - return handler.ProgressEvent{ - OperationStatus: handler.Success, - Message: "Read Completed", - ResourceModel: model}, nil -} - -func ReadResponseModelGeneration(pe *admin20231115014.DataLakeIngestionPipeline) (model *Model) { - if pe != nil { - source := Source{ - Type: pe.Source.Type, - ClusterName: pe.Source.ClusterName, - CollectionName: pe.Source.CollectionName, - DatabaseName: pe.Source.DatabaseName, - GroupId: pe.Source.GroupId, - } - - partitionArr := []PartitionFields{} - sink := Sink{} - - if pe.Sink.PartitionFields != nil { - partitionFields := pe.Sink.GetPartitionFields() - for i := range partitionFields { - partitionField := PartitionFields{ - FieldName: &partitionFields[i].FieldName, - Order: &partitionFields[i].Order, - } - partitionArr = append(partitionArr, partitionField) - } - sink.Type = pe.Sink.Type - sink.MetadataProvider = pe.Sink.MetadataProvider - sink.MetadataRegion = pe.Sink.MetadataRegion - sink.PartitionFields = partitionArr - } - transformationsArr := []Transformations{} - if pe.Transformations != nil { - transformations := pe.GetTransformations() - for i := range transformations { - transformations := Transformations{ - Field: transformations[i].Field, - Type: transformations[i].Type, - } - transformationsArr = append(transformationsArr, transformations) - } - } - - createdStr := pe.CreatedDate.Format(time.RFC3339) - lastUpdatedStr := pe.LastUpdatedDate.Format(time.RFC3339) - - models := Model{ - ProjectId: pe.GroupId, - Name: pe.Name, - Id: pe.Id, - CreatedDate: &createdStr, - LastUpdatedDate: &lastUpdatedStr, - Sink: &sink, - Source: &source, - State: pe.State, - Transformations: transformationsArr, - } - return &models - } - return nil -} - -func Update(req handler.Request, prevModel *Model, currentModel *Model) (handler.ProgressEvent, error) { - setup() - - modelValidation := validator.ValidateModel(UpdateRequiredFields, currentModel) - if modelValidation != nil { - return *modelValidation, nil - } - - if currentModel.Profile == nil || *currentModel.Profile == "" { - currentModel.Profile = aws.String(profile.DefaultProfile) - } - - client, peErr := util.NewAtlasClient(&req, currentModel.Profile) - if peErr != nil { - return *peErr, nil - } - - groupID := *currentModel.ProjectId - pipelineName := *currentModel.Name - dataLakeIntegrationPipeline := generateDataLakeIntegrationPipeline(currentModel) - - pe, response, err := client.Atlas20231115014.DataLakePipelinesApi.UpdatePipeline(ctx.Background(), groupID, pipelineName, dataLakeIntegrationPipeline).Execute() - - if err != nil { - if response.StatusCode == http.StatusBadRequest { - return progress_events.GetFailedEventByCode(fmt.Sprintf("Error during execution : %s", err.Error()), - cloudformation.HandlerErrorCodeAlreadyExists), nil - } - return handleError(response, err) - } - - model := ReadResponseModelGeneration(pe) - if model == nil { - errorMsg := "Response model from the API is empty or nil " - return progress_events.GetFailedEventByResponse(errorMsg, response), nil - } - model.Profile = currentModel.Profile - return handler.ProgressEvent{ - OperationStatus: handler.Success, - Message: "Update Completed", - ResourceModel: model}, nil -} - -func Delete(req handler.Request, prevModel *Model, currentModel *Model) (handler.ProgressEvent, error) { - setup() - - modelValidation := validator.ValidateModel(DeleteRequiredFields, currentModel) - if modelValidation != nil { - return *modelValidation, nil - } - - if currentModel.Profile == nil || *currentModel.Profile == "" { - currentModel.Profile = aws.String(profile.DefaultProfile) - } - - client, peErr := util.NewAtlasClient(&req, currentModel.Profile) - if peErr != nil { - return *peErr, nil - } - - groupID := *currentModel.ProjectId - pipelineName := *currentModel.Name - - _, response, err := client.Atlas20231115014.DataLakePipelinesApi.DeletePipeline(ctx.Background(), groupID, pipelineName).Execute() - - if err != nil { - return handleError(response, err) - } - return handler.ProgressEvent{ - OperationStatus: handler.Success, - Message: "Delete Completed", - ResourceModel: nil}, nil -} - -func List(req handler.Request, prevModel *Model, currentModel *Model) (handler.ProgressEvent, error) { - setup() - - modelValidation := validator.ValidateModel(ListRequiredFields, currentModel) - if modelValidation != nil { - return *modelValidation, nil - } - - if currentModel.Profile == nil || *currentModel.Profile == "" { - currentModel.Profile = aws.String(profile.DefaultProfile) - } - - client, peErr := util.NewAtlasClient(&req, currentModel.Profile) - if peErr != nil { - return *peErr, nil - } - - groupID := *currentModel.ProjectId - pe, response, err := client.Atlas20231115014.DataLakePipelinesApi.ListPipelines(ctx.Background(), groupID).Execute() - - if err != nil { - return handleError(response, err) - } - - var list = make([]interface{}, 0) - for ind := range pe { - model := ReadResponseModelGeneration(&pe[ind]) - if model == nil { - errorMsg := "Response model from the API is empty or nil " - return progress_events.GetFailedEventByResponse(errorMsg, response), nil - } - model.Profile = currentModel.Profile - list = append(list, *model) - } - - return handler.ProgressEvent{ - OperationStatus: handler.Success, - Message: "List Complete", - ResourceModels: list}, nil -} - -func handleError(response *http.Response, err error) (handler.ProgressEvent, error) { - if response.StatusCode == http.StatusConflict { - return handler.ProgressEvent{ - OperationStatus: handler.Failed, - Message: err.Error(), - HandlerErrorCode: cloudformation.HandlerErrorCodeAlreadyExists}, nil - } - return progress_events.GetFailedEventByResponse(fmt.Sprintf("Error during execution : %s", err.Error()), response), nil -} diff --git a/cfn-resources/data-lake-pipeline/docs/README.md b/cfn-resources/data-lake-pipeline/docs/README.md deleted file mode 100644 index 7b7c8f36b..000000000 --- a/cfn-resources/data-lake-pipeline/docs/README.md +++ /dev/null @@ -1,132 +0,0 @@ -# MongoDB::Atlas::DataLakePipeline - -Data Lake is deprecated. As of September 2024, Data Lake is deprecated. If you use Data Lake, you should migrate to alternative solutions before the service is removed. To learn more, see . This resource returns, adds, edits, and removes data lake pipelines. - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "Type" : "MongoDB::Atlas::DataLakePipeline",
-    "Properties" : {
-        "Profile" : String,
-        "ProjectId" : String,
-        "State" : String,
-        "Name" : String,
-        "Sink" : sink,
-        "Source" : source,
-        "Transformations" : [ transformations, ... ]
-    }
-}
-
- -### YAML - -
-Type: MongoDB::Atlas::DataLakePipeline
-Properties:
-    Profile: String
-    ProjectId: String
-    State: String
-    Name: String
-    Sink: sink
-    Source: source
-    Transformations: 
-      - transformations
-
- -## Properties - -#### Profile - -Profile used to provide credentials information, (a secret with the cfn/atlas/profile/{Profile}, is required), if not provided default is used - -_Required_: No - -_Type_: String - -_Update requires_: [Replacement](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-replacement) - -#### ProjectId - -Unique 24-hexadecimal digit string that identifies your project. - -_Required_: Yes - -_Type_: String - -_Update requires_: [Replacement](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-replacement) - -#### State - -State of the Data Lake Pipeline. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Name - -Name of this Data Lake Pipeline. - -_Required_: Yes - -_Type_: String - -_Update requires_: [Replacement](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-replacement) - -#### Sink - -Ingestion destination of a Data Lake Pipeline. - -_Required_: Yes - -_Type_: sink - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Source - -Ingestion destination of a Data Lake Pipeline. - -_Required_: No - -_Type_: source - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Transformations - -Ingestion destination of a Data Lake Pipeline. - -_Required_: Yes - -_Type_: List of transformations - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -## Return Values - -### Fn::GetAtt - -The `Fn::GetAtt` intrinsic function returns a value for a specified attribute of this type. The following are the available attributes and sample return values. - -For more information about using the `Fn::GetAtt` intrinsic function, see [Fn::GetAtt](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/intrinsic-function-reference-getatt.html). - -#### CreatedDate - -Timestamp that indicates when the Data Lake Pipeline was created. - -#### Id - -Unique 24-hexadecimal digit string that identifies your project. - -#### LastUpdatedDate - -Timestamp that indicates the last time that the Data Lake Pipeline was updated. - diff --git a/cfn-resources/data-lake-pipeline/docs/partitionfields.md b/cfn-resources/data-lake-pipeline/docs/partitionfields.md deleted file mode 100644 index e840e20c3..000000000 --- a/cfn-resources/data-lake-pipeline/docs/partitionfields.md +++ /dev/null @@ -1,46 +0,0 @@ -# MongoDB::Atlas::DataLakePipeline partitionFields - -Ordered fields used to physically organize data in the destination. - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "FieldName" : String,
-    "Order" : Integer
-}
-
- -### YAML - -
-FieldName: String
-Order: Integer
-
- -## Properties - -#### FieldName - -Human-readable label that identifies the field name used to partition data. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Order - -Sequence in which MongoDB Cloud slices the collection data to create partitions. The resource expresses this sequence starting with zero. - -_Required_: No - -_Type_: Integer - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/data-lake-pipeline/docs/sink.md b/cfn-resources/data-lake-pipeline/docs/sink.md deleted file mode 100644 index 82e6ea381..000000000 --- a/cfn-resources/data-lake-pipeline/docs/sink.md +++ /dev/null @@ -1,73 +0,0 @@ -# MongoDB::Atlas::DataLakePipeline sink - -Ingestion destination of a Data Lake Pipeline. - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "Type" : String,
-    "MetadataProvider" : String,
-    "MetadataRegion" : String,
-    "PartitionFields" : [ partitionFields, ... ]
-}
-
- -### YAML - -
-Type: String
-MetadataProvider: String
-MetadataRegion: String
-PartitionFields: 
-      - partitionFields
-
- -## Properties - -#### Type - -Type of ingestion destination of this Data Lake Pipeline. - -_Required_: No - -_Type_: String - -_Allowed Values_: DLS - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### MetadataProvider - -Target cloud provider for this Data Lake Pipeline. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### MetadataRegion - -Target cloud provider region for this Data Lake Pipeline. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### PartitionFields - -Ordered fields used to physically organize data in the destination. - -_Required_: No - -_Type_: List of partitionFields - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/data-lake-pipeline/docs/source.md b/cfn-resources/data-lake-pipeline/docs/source.md deleted file mode 100644 index 4f0dafe96..000000000 --- a/cfn-resources/data-lake-pipeline/docs/source.md +++ /dev/null @@ -1,90 +0,0 @@ -# MongoDB::Atlas::DataLakePipeline source - -Ingestion destination of a Data Lake Pipeline. - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "Type" : String,
-    "ClusterName" : String,
-    "CollectionName" : String,
-    "DatabaseName" : String,
-    "GroupId" : String
-}
-
- -### YAML - -
-Type: String
-ClusterName: String
-CollectionName: String
-DatabaseName: String
-GroupId: String
-
- -## Properties - -#### Type - -Type of ingestion source of this Data Lake Pipeline. - -_Required_: No - -_Type_: String - -_Allowed Values_: ON_DEMAND_CPS | PERIODIC_CPS - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### ClusterName - -Human-readable name that identifies the cluster. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### CollectionName - -Human-readable name that identifies the collection. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### DatabaseName - -Human-readable name that identifies the database. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### GroupId - -Unique 24-hexadecimal character string that identifies the project. - -_Required_: No - -_Type_: String - -_Minimum Length_: 24 - -_Maximum Length_: 24 - -_Pattern_: ^([a-f0-9]{24})$ - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/data-lake-pipeline/docs/transformations.md b/cfn-resources/data-lake-pipeline/docs/transformations.md deleted file mode 100644 index 720b21c75..000000000 --- a/cfn-resources/data-lake-pipeline/docs/transformations.md +++ /dev/null @@ -1,46 +0,0 @@ -# MongoDB::Atlas::DataLakePipeline transformations - -Ordered fields used to physically organize data in the destination. - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "Field" : String,
-    "Type" : String
-}
-
- -### YAML - -
-Field: String
-Type: String
-
- -## Properties - -#### Field - -Key in the document. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Type - -Type of transformation applied during the export of the namespace in a Data Lake Pipeline. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/data-lake-pipeline/mongodb-atlas-datalakepipeline.json b/cfn-resources/data-lake-pipeline/mongodb-atlas-datalakepipeline.json deleted file mode 100644 index a5043949b..000000000 --- a/cfn-resources/data-lake-pipeline/mongodb-atlas-datalakepipeline.json +++ /dev/null @@ -1,203 +0,0 @@ -{ - "typeName": "MongoDB::Atlas::DataLakePipeline", - "description": "Data Lake is deprecated. As of September 2024, Data Lake is deprecated. If you use Data Lake, you should migrate to alternative solutions before the service is removed. To learn more, see . This resource returns, adds, edits, and removes data lake pipelines.", - "sourceUrl": "https://github.com/mongodb/mongodbatlas-cloudformation-resources/tree/master/cfn-resources/data-lake-pipeline", - "definitions": { - "sink": { - "type": "object", - "description": "Ingestion destination of a Data Lake Pipeline.", - "properties": { - "Type": { - "type": "string", - "description": "Type of ingestion destination of this Data Lake Pipeline.", - "enum": [ - "DLS" - ] - }, - "MetadataProvider": { - "type": "string", - "description": "Target cloud provider for this Data Lake Pipeline." - }, - "MetadataRegion": { - "type": "string", - "description": "Target cloud provider region for this Data Lake Pipeline." - }, - "PartitionFields": { - "type": "array", - "insertionOrder": false, - "description": "Ordered fields used to physically organize data in the destination.", - "items": { - "$ref": "#/definitions/partitionFields" - } - } - }, - "additionalProperties": false - }, - "source": { - "type": "object", - "description": "Ingestion destination of a Data Lake Pipeline.", - "properties": { - "Type": { - "type": "string", - "enum": [ - "ON_DEMAND_CPS", - "PERIODIC_CPS" - ], - "description": "Type of ingestion source of this Data Lake Pipeline." - }, - "ClusterName": { - "type": "string", - "description": "Human-readable name that identifies the cluster." - }, - "CollectionName": { - "type": "string", - "description": "Human-readable name that identifies the collection." - }, - "DatabaseName": { - "type": "string", - "description": "Human-readable name that identifies the database." - }, - "GroupId": { - "type": "string", - "description": "Unique 24-hexadecimal character string that identifies the project.", - "maxLength": 24, - "minLength": 24, - "pattern": "^([a-f0-9]{24})$" - } - }, - "additionalProperties": false - }, - "partitionFields": { - "type": "object", - "description": "Ordered fields used to physically organize data in the destination.", - "properties": { - "FieldName": { - "type": "string", - "description": "Human-readable label that identifies the field name used to partition data." - }, - "Order": { - "type": "integer", - "description": "Sequence in which MongoDB Cloud slices the collection data to create partitions. The resource expresses this sequence starting with zero." - } - }, - "additionalProperties": false - }, - "transformations": { - "type": "object", - "insertionOrder": false, - "description": "Ordered fields used to physically organize data in the destination.", - "properties": { - "Field": { - "type": "string", - "description": "Key in the document." - }, - "Type": { - "type": "string", - "description": "Type of transformation applied during the export of the namespace in a Data Lake Pipeline." - } - }, - "additionalProperties": false - } - }, - "properties": { - "Id": { - "type": "string", - "description": "Unique 24-hexadecimal digit string that identifies your project.", - "maxLength": 24, - "minLength": 24, - "pattern": "^([a-f0-9]{24})$" - }, - "Profile": { - "type": "string", - "description": "Profile used to provide credentials information, (a secret with the cfn/atlas/profile/{Profile}, is required), if not provided default is used", - "default": "default" - }, - "ProjectId": { - "type": "string", - "description": "Unique 24-hexadecimal digit string that identifies your project." - }, - "State": { - "type": "string", - "description": "State of the Data Lake Pipeline." - }, - "Name": { - "type": "string", - "description": "Name of this Data Lake Pipeline." - }, - "CreatedDate": { - "type": "string", - "description": "Timestamp that indicates when the Data Lake Pipeline was created." - }, - "LastUpdatedDate": { - "type": "string", - "description": "Timestamp that indicates the last time that the Data Lake Pipeline was updated." - }, - "Sink": { - "$ref": "#/definitions/sink" - }, - "Source": { - "$ref": "#/definitions/source" - }, - "Transformations": { - "type": "array", - "description": "Ingestion destination of a Data Lake Pipeline.", - "items": { - "$ref": "#/definitions/transformations" - } - } - }, - "additionalProperties": false, - "required": [ - "ProjectId", - "Transformations", - "Sink", - "Name" - ], - "createOnlyProperties": [ - "/properties/Name", - "/properties/Source/ClusterName", - "/properties/Profile", - "/properties/ProjectId" - ], - "readOnlyProperties": [ - "/properties/CreatedDate", - "/properties/Id", - "/properties/LastUpdatedDate" - ], - "primaryIdentifier": [ - "/properties/ProjectId", - "/properties/Name", - "/properties/Profile" - ], - "handlers": { - "create": { - "permissions": [ - "secretsmanager:GetSecretValue" - ] - }, - "read": { - "permissions": [ - "secretsmanager:GetSecretValue" - ] - }, - "update": { - "permissions": [ - "secretsmanager:GetSecretValue" - ] - }, - "delete": { - "permissions": [ - "secretsmanager:GetSecretValue" - ] - }, - "list": { - "permissions": [ - "secretsmanager:GetSecretValue" - ] - } - }, - "documentationUrl": "https://github.com/mongodb/mongodbatlas-cloudformation-resources/blob/master/cfn-resources/data-lake-pipeline/README.md", - "tagging": { - "taggable": false - } -} diff --git a/cfn-resources/data-lake-pipeline/resource-role.yaml b/cfn-resources/data-lake-pipeline/resource-role.yaml deleted file mode 100644 index 40d6a44db..000000000 --- a/cfn-resources/data-lake-pipeline/resource-role.yaml +++ /dev/null @@ -1,38 +0,0 @@ -AWSTemplateFormatVersion: "2010-09-09" -Description: > - This CloudFormation template creates a role assumed by CloudFormation - during CRUDL operations to mutate resources on behalf of the customer. - -Resources: - ExecutionRole: - Type: AWS::IAM::Role - Properties: - MaxSessionDuration: 8400 - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: resources.cloudformation.amazonaws.com - Action: sts:AssumeRole - Condition: - StringEquals: - aws:SourceAccount: - Ref: AWS::AccountId - StringLike: - aws:SourceArn: - Fn::Sub: arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:type/resource/MongoDB-Atlas-DataLakePipeline/* - Path: "/" - Policies: - - PolicyName: ResourceTypePolicy - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - "secretsmanager:GetSecretValue" - Resource: "*" -Outputs: - ExecutionRoleArn: - Value: - Fn::GetAtt: ExecutionRole.Arn diff --git a/cfn-resources/data-lake-pipeline/template.yml b/cfn-resources/data-lake-pipeline/template.yml deleted file mode 100644 index 68a55c1da..000000000 --- a/cfn-resources/data-lake-pipeline/template.yml +++ /dev/null @@ -1,29 +0,0 @@ -AWSTemplateFormatVersion: "2010-09-09" -Transform: AWS::Serverless-2016-10-31 -Description: AWS SAM template for the MongoDB::Atlas::DataLakePipeline resource type - -Globals: - Function: - Timeout: 180 # docker start-up times can be long for SAM CLI - MemorySize: 256 - -Resources: - TypeFunction: - Type: AWS::Serverless::Function - Properties: - Handler: bootstrap - Runtime: provided.al2 - CodeUri: bin/ - - TestEntrypoint: - Type: AWS::Serverless::Function - Properties: - Handler: bootstrap - Runtime: provided.al2 - CodeUri: bin/ - Environment: - Variables: - MODE: Test - LOG_LEVEL: debug - MONGODB_ATLAS_BASE_URL: - diff --git a/cfn-resources/data-lake-pipeline/test/cfn-test-create-inputs.sh b/cfn-resources/data-lake-pipeline/test/cfn-test-create-inputs.sh deleted file mode 100755 index c06acd2fc..000000000 --- a/cfn-resources/data-lake-pipeline/test/cfn-test-create-inputs.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2023 MongoDB Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# cfn-test-create-inputs.sh -# -# This tool generates json files in the inputs/ for `cfn test`. -# - -set -o errexit -set -o nounset -set -o pipefail - -function usage { - echo "usage:$0 " - echo "Creates a new Data lake pipeline with the cluster details for the test" -} - -if [ "$#" -ne 2 ]; then usage; fi -if [[ "$*" == help ]]; then usage; fi - -rm -rf inputs -mkdir inputs - -#set profile -profile="default" -if [ ${MONGODB_ATLAS_PROFILE+x} ]; then - echo "profile set to ${MONGODB_ATLAS_PROFILE}" - profile=${MONGODB_ATLAS_PROFILE} -fi - -projectName="${1}" -projectId=$(atlas projects list --output json | jq --arg NAME "${projectName}" -r '.results[] | select(.name==$NAME) | .id') -if [ -z "$projectId" ]; then - projectId=$(atlas projects create "${projectName}" --output=json | jq -r '.id') - - echo -e "Created project \"${projectName}\" with id: ${projectId}\n" -else - echo -e "FOUND project \"${projectName}\" with id: ${projectId}\n" -fi - -echo "Check if a project is created $projectId" - -clusterName="${projectName}" - -atlas clusters create "${clusterName}" --projectId "${projectId}" --backup --provider AWS --region US_EAST_1 --members 3 --tier M10 --mdbVersion 5.0 --diskSizeGB 10 --output=json -atlas clusters watch "${clusterName}" --projectId "${projectId}" -echo -e "Created Cluster \"${clusterName}\"" - -jq --arg clusterName "$clusterName" \ - --arg projectId "$projectId" \ - --arg profile "$profile" \ - '.Profile?|=$profile | .Source.ClusterName?|=$clusterName | .ProjectId?|=$projectId ' \ - "$(dirname "$0")/inputs_1_create.template.json" >"inputs/inputs_1_create.json" - -jq --arg clusterName "$clusterName" \ - --arg projectId "$projectId" \ - --arg profile "$profile" \ - '.Profile?|=$profile | .Source.ClusterName?|=$clusterName | .ProjectId?|=$projectId ' \ - "$(dirname "$0")/inputs_1_update.template.json" >"inputs/inputs_1_update.json" diff --git a/cfn-resources/data-lake-pipeline/test/cfn-test-delete-inputs.sh b/cfn-resources/data-lake-pipeline/test/cfn-test-delete-inputs.sh deleted file mode 100755 index 7e58e31e8..000000000 --- a/cfn-resources/data-lake-pipeline/test/cfn-test-delete-inputs.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2023 MongoDB Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# cfn-test-create-inputs.sh -# -# This tool deletes the mongodb resources used for `cfn test` as inputs. -# - -set -o errexit -set -o nounset -set -o pipefail - -function usage { - echo "usage:$0 " -} - -projectId=$(jq -r '.ProjectId' ./inputs/inputs_1_create.json) - -clusterName=$(jq -r '.Source.ClusterName' ./inputs/inputs_1_create.json) - -#delete Cluster -if atlas clusters delete "$clusterName" --projectId "${projectId}" --force; then - echo "$clusterName cluster deletion OK" -else - (echo "Failed cleaning cluster:$clusterName" && exit 1) -fi - -status="DELETING" -echo "Waiting for cluster to get deleted" -while [[ "${status}" == "DELETING" ]]; do - sleep 60 - if atlas clusters describe "${clusterName}" --projectId "${projectId}"; then - status=$(atlas clusters describe "${clusterName}" --projectId "${projectId}" --output=json | jq -r '.stateName') - else - status="DELETED" - fi - echo "status: ${status}" -done - - -#delete Project -if atlas projects delete "$projectId" --force; then - echo "$projectId is deleted" -else - (echo "Failed cleaning project:$projectId" && exit 1) -fi - - - - - diff --git a/cfn-resources/data-lake-pipeline/test/contract-testing/cfn-test-create-inputs.sh b/cfn-resources/data-lake-pipeline/test/contract-testing/cfn-test-create-inputs.sh deleted file mode 100755 index 6e05ae4d5..000000000 --- a/cfn-resources/data-lake-pipeline/test/contract-testing/cfn-test-create-inputs.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# cfn-test-create-inputs.sh -# -# This tool generates json files in the inputs/ for `cfn test`. -# - -set -o errexit -set -o nounset -set -o pipefail - -# setting projectName -projectName="cfn-bot-datalake-pipeline-$((1 + RANDOM % 10000))" - -./test/cfn-test-create-inputs.sh $projectName diff --git a/cfn-resources/data-lake-pipeline/test/inputs_1_create.template.json b/cfn-resources/data-lake-pipeline/test/inputs_1_create.template.json deleted file mode 100644 index fe7a50b71..000000000 --- a/cfn-resources/data-lake-pipeline/test/inputs_1_create.template.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "ProjectId": "", - "Profile": "default", - "Name": "testdatalakepipeline1", - "Sink": { - "PartitionFields": [ - { - "FieldName": "testpartition2", - "Order": "0" - } - ] - }, - "Source": { - "ClusterName": "", - "CollectionName": "testcollection3", - "DatabaseName": "testdatabase4" - }, - "Transformations": [ - { - "Field": "testtransform4", - "Type": "EXCLUDE" - } - ] -} \ No newline at end of file diff --git a/cfn-resources/data-lake-pipeline/test/inputs_1_update.template.json b/cfn-resources/data-lake-pipeline/test/inputs_1_update.template.json deleted file mode 100644 index 319095de5..000000000 --- a/cfn-resources/data-lake-pipeline/test/inputs_1_update.template.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "ProjectId": "", - "Profile": "default", - "Name": "testdatalakepipeline1", - "Sink": { - "PartitionFields": [ - { - "FieldName": "testpartition2", - "Order": "0" - } - ] - }, - "Source": { - "ClusterName": "", - "CollectionName": "testcollection3", - "DatabaseName": "testdatabase4" - }, - "Transformations": [ - { - "Field": "testtransform50", - "Type": "EXCLUDE" - } - ] -} \ No newline at end of file diff --git a/cfn-resources/datalakes/.rpdk-config b/cfn-resources/datalakes/.rpdk-config deleted file mode 100644 index bc9e11074..000000000 --- a/cfn-resources/datalakes/.rpdk-config +++ /dev/null @@ -1,22 +0,0 @@ -{ - "artifact_type": "RESOURCE", - "typeName": "MongoDB::Atlas::DataLakes", - "language": "go", - "runtime": "provided.al2", - "entrypoint": "bootstrap", - "testEntrypoint": "bootstrap", - "settings": { - "version": false, - "subparser_name": null, - "verbose": 0, - "force": false, - "type_name": null, - "artifact_type": null, - "endpoint_url": null, - "region": null, - "target_schemas": [], - "import_path": "github.com/mongodb/mongodbatlas-cloudformation-resources/datalakes", - "protocolVersion": "2.0.0", - "pluginVersion": "2.0.4" - } -} diff --git a/cfn-resources/datalakes/Makefile b/cfn-resources/datalakes/Makefile deleted file mode 100644 index 6df18d210..000000000 --- a/cfn-resources/datalakes/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -.PHONY: build test clean -tags=logging callback metrics scheduler -cgo=0 -goos=linux -goarch=amd64 -CFNREP_GIT_SHA?=$(shell git rev-parse HEAD) -ldXflags=-s -w -X github.com/mongodb/mongodbatlas-cloudformation-resources/util.defaultLogLevel=info -X github.com/mongodb/mongodbatlas-cloudformation-resources/version.Version=${CFNREP_GIT_SHA} -ldXflagsD=-X github.com/mongodb/mongodbatlas-cloudformation-resources/util.defaultLogLevel=debug -X github.com/mongodb/mongodbatlas-cloudformation-resources/version.Version=${CFNREP_GIT_SHA} - -build: - cfn generate - env GOOS=$(goos) CGO_ENABLED=$(cgo) GOARCH=$(goarch) go build -ldflags="$(ldXflags)" -tags="$(tags)" -o bin/bootstrap cmd/main.go - -debug: - cfn generate - env GOOS=$(goos) CGO_ENABLED=$(cgo) GOARCH=$(goarch) go build -ldflags="$(ldXflagsD)" -tags="$(tags)" -o bin/bootstrap cmd/main.go - -clean: - rm -rf bin diff --git a/cfn-resources/datalakes/README.md b/cfn-resources/datalakes/README.md deleted file mode 100644 index d451fa9d0..000000000 --- a/cfn-resources/datalakes/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# MongoDB::Atlas::DataLakes - -## Description -Resource for managing [Federated Database Instances](https://www.mongodb.com/docs/api/doc/atlas-admin-api-v2/group/endpoint-data-federation). - -## Requirements - -Set up an AWS profile to securely give CloudFormation access to your Atlas credentials. -For instructions on setting up a profile, [see here](/README.md#mongodb-atlas-api-keys-credential-management). - -## Attributes & Parameters - -See the [resource docs](docs/README.md). - -## CloudFormation Examples - -See the examples [CFN Template](/examples/data-lakes/datalake.json) for example resource. diff --git a/cfn-resources/datalakes/cmd/main.go b/cfn-resources/datalakes/cmd/main.go deleted file mode 100644 index 594cbd940..000000000 --- a/cfn-resources/datalakes/cmd/main.go +++ /dev/null @@ -1,85 +0,0 @@ -// Code generated by 'cfn generate', changes will be undone by the next invocation. DO NOT EDIT. -package main - -import ( - "errors" - "fmt" - "log" - - "github.com/aws-cloudformation/cloudformation-cli-go-plugin/cfn" - "github.com/aws-cloudformation/cloudformation-cli-go-plugin/cfn/handler" - "github.com/mongodb/mongodbatlas-cloudformation-resources/datalakes/cmd/resource" -) - -// Handler is a container for the CRUDL actions exported by resources -type Handler struct{} - -// Create wraps the related Create function exposed by the resource code -func (r *Handler) Create(req handler.Request) handler.ProgressEvent { - return wrap(req, resource.Create) -} - -// Read wraps the related Read function exposed by the resource code -func (r *Handler) Read(req handler.Request) handler.ProgressEvent { - return wrap(req, resource.Read) -} - -// Update wraps the related Update function exposed by the resource code -func (r *Handler) Update(req handler.Request) handler.ProgressEvent { - return wrap(req, resource.Update) -} - -// Delete wraps the related Delete function exposed by the resource code -func (r *Handler) Delete(req handler.Request) handler.ProgressEvent { - return wrap(req, resource.Delete) -} - -// List wraps the related List function exposed by the resource code -func (r *Handler) List(req handler.Request) handler.ProgressEvent { - return wrap(req, resource.List) -} - -// main is the entry point of the application. -func main() { - cfn.Start(&Handler{}) -} - -type handlerFunc func(handler.Request, *resource.Model, *resource.Model) (handler.ProgressEvent, error) - -func wrap(req handler.Request, f handlerFunc) (response handler.ProgressEvent) { - defer func() { - // Catch any panics and return a failed ProgressEvent - if r := recover(); r != nil { - err, ok := r.(error) - if !ok { - err = errors.New(fmt.Sprint(r)) - } - - log.Printf("Trapped error in handler: %v", err) - - response = handler.NewFailedEvent(err) - } - }() - - // Populate the previous model - prevModel := &resource.Model{} - if err := req.UnmarshalPrevious(prevModel); err != nil { - log.Printf("Error unmarshaling prev model: %v", err) - return handler.NewFailedEvent(err) - } - - // Populate the current model - currentModel := &resource.Model{} - if err := req.Unmarshal(currentModel); err != nil { - log.Printf("Error unmarshaling model: %v", err) - return handler.NewFailedEvent(err) - } - - response, err := f(req, prevModel, currentModel) - if err != nil { - log.Printf("Error returned from handler function: %v", err) - return handler.NewFailedEvent(err) - } - - return response -} diff --git a/cfn-resources/datalakes/cmd/resource/model.go b/cfn-resources/datalakes/cmd/resource/model.go deleted file mode 100644 index f862bb8df..000000000 --- a/cfn-resources/datalakes/cmd/resource/model.go +++ /dev/null @@ -1,89 +0,0 @@ -// Code generated by 'cfn generate', changes will be undone by the next invocation. DO NOT EDIT. -// Updates to this type are made my editing the schema file and executing the 'generate' command. -package resource - -// Model is autogenerated from the json schema -type Model struct { - CloudProviderConfig *DataLakeCloudProviderConfigView `json:",omitempty"` - DataProcessRegion *DataLakeDataProcessRegionView `json:",omitempty"` - EndDate *float64 `json:",omitempty"` - ProjectId *string `json:",omitempty"` - Profile *string `json:",omitempty"` - SkipRoleValidation *bool `json:",omitempty"` - StartDate *float64 `json:",omitempty"` - Storage *DataLakeStorageView `json:",omitempty"` - TenantName *string `json:",omitempty"` - Hostnames []string `json:",omitempty"` - State *string `json:",omitempty"` -} - -// DataLakeCloudProviderConfigView is autogenerated from the json schema -type DataLakeCloudProviderConfigView struct { - Aws *DataLakeAWSCloudProviderConfigView `json:",omitempty"` -} - -// DataLakeAWSCloudProviderConfigView is autogenerated from the json schema -type DataLakeAWSCloudProviderConfigView struct { - ExternalId *string `json:",omitempty"` - IamAssumedRoleARN *string `json:",omitempty"` - IamUserARN *string `json:",omitempty"` - RoleId *string `json:",omitempty"` - TestS3Bucket *string `json:",omitempty"` -} - -// DataLakeDataProcessRegionView is autogenerated from the json schema -type DataLakeDataProcessRegionView struct { - CloudProvider *string `json:",omitempty"` - Region *string `json:",omitempty"` -} - -// DataLakeStorageView is autogenerated from the json schema -type DataLakeStorageView struct { - Databases []DataLakeDatabaseView `json:",omitempty"` - Stores []StoreDetail `json:",omitempty"` -} - -// DataLakeDatabaseView is autogenerated from the json schema -type DataLakeDatabaseView struct { - Collections []DataLakeDatabaseCollectionView `json:",omitempty"` - MaxWildcardCollections *int `json:",omitempty"` - Name *string `json:",omitempty"` - Views []DataLakeViewView `json:",omitempty"` -} - -// DataLakeDatabaseCollectionView is autogenerated from the json schema -type DataLakeDatabaseCollectionView struct { - DataSources []DataLakeDatabaseDataSourceView `json:",omitempty"` - Name *string `json:",omitempty"` -} - -// DataLakeDatabaseDataSourceView is autogenerated from the json schema -type DataLakeDatabaseDataSourceView struct { - AllowInsecure *bool `json:",omitempty"` - Collection *string `json:",omitempty"` - CollectionRegex *string `json:",omitempty"` - Database *string `json:",omitempty"` - DefaultFormat *string `json:",omitempty"` - Path *string `json:",omitempty"` - StoreName *string `json:",omitempty"` - Urls []string `json:",omitempty"` -} - -// DataLakeViewView is autogenerated from the json schema -type DataLakeViewView struct { - Name *string `json:",omitempty"` - Pipeline *string `json:",omitempty"` - Source *string `json:",omitempty"` -} - -// StoreDetail is autogenerated from the json schema -type StoreDetail struct { - Name *string `json:",omitempty"` - Provider *string `json:",omitempty"` - Region *string `json:",omitempty"` - Bucket *string `json:",omitempty"` - Prefix *string `json:",omitempty"` - Delimiter *string `json:",omitempty"` - IncludeTags *bool `json:",omitempty"` - AdditionalStorageClasses []string `json:",omitempty"` -} diff --git a/cfn-resources/datalakes/cmd/resource/resource.go b/cfn-resources/datalakes/cmd/resource/resource.go deleted file mode 100644 index fb114e2f2..000000000 --- a/cfn-resources/datalakes/cmd/resource/resource.go +++ /dev/null @@ -1,437 +0,0 @@ -// Copyright 2023 MongoDB Inc -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package resource - -import ( - "context" - "fmt" - "net/http" - - "github.com/aws-cloudformation/cloudformation-cli-go-plugin/cfn/handler" - "github.com/aws/aws-sdk-go/service/cloudformation" - "github.com/mongodb/mongodbatlas-cloudformation-resources/util" - "github.com/mongodb/mongodbatlas-cloudformation-resources/util/constants" - "github.com/mongodb/mongodbatlas-cloudformation-resources/util/progressevent" - "github.com/mongodb/mongodbatlas-cloudformation-resources/util/validator" - "github.com/spf13/cast" - admin20231115002 "go.mongodb.org/atlas-sdk/v20231115002/admin" -) - -var RequiredFields = []string{constants.ProjectID, constants.TenantName} -var ListRequiredFields = []string{constants.ProjectID} - -// Create handles the Create event from the Cloudformation service. -func Create(req handler.Request, prevModel *Model, currentModel *Model) (handler.ProgressEvent, error) { - setup() // logger setup - - validationError := validateRequest(RequiredFields, currentModel) - if validationError != nil { - return *validationError, nil - } - - util.SetDefaultProfileIfNotDefined(¤tModel.Profile) - client, peErr := util.NewAtlasClient(&req, currentModel.Profile) - if peErr != nil { - return *peErr, nil - } - - if _, ok := req.CallbackContext["status"]; ok { - sid := req.CallbackContext["project_id"].(string) - currentModel.ProjectId = &sid - return validateProgress(client, currentModel, "ACTIVE") - } - - projectID := *currentModel.ProjectId - dataLakeReq := &admin20231115002.DataLakeTenant{ - CloudProviderConfig: expandCloudProviderConfig(currentModel), - Name: currentModel.TenantName, - } - - dataLake, resp, err := client.Atlas20231115002.DataFederationApi.CreateFederatedDatabase(context.Background(), projectID, dataLakeReq).Execute() - if err != nil { - return progressevent.GetFailedEventByResponse(err.Error(), resp), nil - } - currentModel.ProjectId = dataLake.GroupId - event := handler.ProgressEvent{ - OperationStatus: handler.InProgress, - Message: fmt.Sprintf("Create cloud provider snapshots : %s", *dataLake.State), - ResourceModel: currentModel, - CallbackDelaySeconds: 65, - CallbackContext: map[string]interface{}{ - "status": dataLake.State, - "project_id": currentModel.ProjectId, - }, - } - return event, nil -} - -// Read handles the Read event from the Cloudformation service. -func Read(req handler.Request, prevModel *Model, currentModel *Model) (handler.ProgressEvent, error) { - setup() // logger setup - - if validationError := validateRequest(RequiredFields, currentModel); validationError != nil { - return *validationError, nil - } - - util.SetDefaultProfileIfNotDefined(¤tModel.Profile) - client, peErr := util.NewAtlasClient(&req, currentModel.Profile) - if peErr != nil { - return *peErr, nil - } - - projectID := *currentModel.ProjectId - dataLake, resp, err := client.Atlas20231115002.DataFederationApi.GetFederatedDatabase(context.Background(), projectID, *currentModel.TenantName).Execute() - if err != nil { - if apiError, ok := admin20231115002.AsError(err); ok && *apiError.Error == http.StatusNotFound { - return handler.ProgressEvent{ - OperationStatus: handler.Failed, - Message: "Resource Not Found", - HandlerErrorCode: cloudformation.HandlerErrorCodeNotFound}, nil - } - - return progressevent.GetFailedEventByResponse(err.Error(), resp), nil - } - - readModel := convertToModel(dataLake, currentModel) - return handler.ProgressEvent{ - OperationStatus: handler.Success, - Message: "Read Complete", - ResourceModel: readModel, - }, nil -} - -// Update handles the Update event from the Cloudformation service. -func Update(req handler.Request, prevModel *Model, currentModel *Model) (handler.ProgressEvent, error) { - setup() // logger setup - - // Validate required fields in the request - if validationError := validateRequest(RequiredFields, currentModel); validationError != nil { - return *validationError, nil - } - - util.SetDefaultProfileIfNotDefined(¤tModel.Profile) - client, peErr := util.NewAtlasClient(&req, currentModel.Profile) - if peErr != nil { - return *peErr, nil - } - - if !isExist(currentModel, client) { - return handler.ProgressEvent{ - OperationStatus: handler.Failed, - Message: "Resource Not Found", - HandlerErrorCode: cloudformation.HandlerErrorCodeNotFound}, nil - } - - projectID := *currentModel.ProjectId - bodyRequest := &admin20231115002.UpdateFederatedDatabaseApiParams{ - GroupId: projectID, - TenantName: *currentModel.TenantName, - SkipRoleValidation: admin20231115002.PtrBool(false), - DataLakeTenant: &admin20231115002.DataLakeTenant{ - CloudProviderConfig: expandCloudProviderConfig(currentModel), - DataProcessRegion: expandDataLakeDataProcessRegion(currentModel.DataProcessRegion), - }, - } - - dataLake, resp, err := client.Atlas20231115002.DataFederationApi.UpdateFederatedDatabaseWithParams(context.Background(), bodyRequest).Execute() - if err != nil { - if resp != nil { - return progressevent.GetFailedEventByResponse(err.Error(), resp), nil - } - - return handler.ProgressEvent{ - OperationStatus: handler.Failed, - Message: "Error in updating the resource", - HandlerErrorCode: cloudformation.HandlerErrorCodeHandlerInternalFailure}, nil - } - - currentModel.ProjectId = dataLake.GroupId - - return handler.ProgressEvent{ - OperationStatus: handler.Success, - Message: "Update Complete", - ResourceModel: currentModel, - }, nil -} - -// Delete handles the Delete event from the Cloudformation service. -func Delete(req handler.Request, prevModel *Model, currentModel *Model) (handler.ProgressEvent, error) { - setup() // logger setup - - // Validate required fields in the request - if validationError := validateRequest(RequiredFields, currentModel); validationError != nil { - return *validationError, nil - } - - util.SetDefaultProfileIfNotDefined(¤tModel.Profile) - client, peErr := util.NewAtlasClient(&req, currentModel.Profile) - if peErr != nil { - return *peErr, nil - } - - if !isExist(currentModel, client) { - return handler.ProgressEvent{ - OperationStatus: handler.Failed, - Message: "Resource Not Found", - HandlerErrorCode: cloudformation.HandlerErrorCodeNotFound}, nil - } - - _, resp, err := client.Atlas20231115002.DataFederationApi.DeleteFederatedDatabase(context.Background(), *currentModel.ProjectId, *currentModel.TenantName).Execute() - if err != nil { - if resp != nil { - return progressevent.GetFailedEventByResponse(err.Error(), resp), nil - } - - return handler.ProgressEvent{ - OperationStatus: handler.Failed, - Message: "Error in deleting the resource", - HandlerErrorCode: cloudformation.HandlerErrorCodeHandlerInternalFailure}, nil - } - - return handler.ProgressEvent{ - OperationStatus: handler.Success, - Message: "Delete Complete", - }, nil -} - -// List handles the list event from the Cloudformation service. -func List(req handler.Request, prevModel *Model, currentModel *Model) (handler.ProgressEvent, error) { - setup() // logger setup - - // Validate required fields in the request - if validationError := validateRequest(ListRequiredFields, currentModel); validationError != nil { - return *validationError, nil - } - - util.SetDefaultProfileIfNotDefined(¤tModel.Profile) - client, peErr := util.NewAtlasClient(&req, currentModel.Profile) - if peErr != nil { - return *peErr, nil - } - - result, resp, err := client.Atlas20231115002.DataFederationApi.ListFederatedDatabases(context.Background(), *currentModel.ProjectId).Execute() - if err != nil { - return progressevent.GetFailedEventByResponse(err.Error(), resp), nil - } - - var models []interface{} - for ind := range result { - models = append(models, convertToModel(&result[ind], currentModel)) - } - - return handler.ProgressEvent{ - OperationStatus: handler.Success, - Message: "List Complete", - ResourceModels: models, - }, nil -} -func flattenAWSBlock(awsConfig *admin20231115002.DataLakeCloudProviderConfig) *DataLakeAWSCloudProviderConfigView { - if awsConfig == nil { - return nil - } - return &DataLakeAWSCloudProviderConfigView{ - RoleId: &awsConfig.Aws.RoleId, - IamAssumedRoleARN: awsConfig.Aws.IamAssumedRoleARN, - IamUserARN: awsConfig.Aws.IamUserARN, - ExternalId: awsConfig.Aws.ExternalId, - TestS3Bucket: &awsConfig.Aws.TestS3Bucket, - } -} - -func flattenDataLakeProcessRegion(processRegion *admin20231115002.DataLakeDataProcessRegion) *DataLakeDataProcessRegionView { - if processRegion != nil && (processRegion.Region != "" || processRegion.CloudProvider != "") { - return &DataLakeDataProcessRegionView{ - CloudProvider: &processRegion.CloudProvider, - Region: &processRegion.Region, - } - } - return nil -} - -func flattenDataLakeStorageDatabases(databases []admin20231115002.DataLakeDatabaseInstance) []DataLakeDatabaseView { - views := make([]DataLakeDatabaseView, len(databases)) - for i := range databases { - views[i] = DataLakeDatabaseView{ - Name: databases[i].Name, - Collections: flattenDataLakeStorageDatabaseCollections(databases[i].Collections), - Views: flattenDataLakeStorageDatabaseViews(databases[i].Views), - MaxWildcardCollections: databases[i].MaxWildcardCollections, - } - } - return views -} - -func flattenDataLakeStorageDatabaseCollections(collections []admin20231115002.DataLakeDatabaseCollection) []DataLakeDatabaseCollectionView { - database := make([]DataLakeDatabaseCollectionView, 0) - for ind := range collections { - database = append(database, DataLakeDatabaseCollectionView{ - Name: collections[ind].Name, - DataSources: flattenDataLakeStorageDatabaseCollectionsDataSources(collections[ind].DataSources), - }) - } - return database -} - -func flattenDataLakeStorageDatabaseCollectionsDataSources(dataSources []admin20231115002.DataLakeDatabaseDataSourceSettings) []DataLakeDatabaseDataSourceView { - database := make([]DataLakeDatabaseDataSourceView, 0) - for ind := range dataSources { - database = append(database, DataLakeDatabaseDataSourceView{ - StoreName: dataSources[ind].StoreName, - DefaultFormat: dataSources[ind].DefaultFormat, - Path: dataSources[ind].Path, - }) - } - return database -} - -func flattenDataLakeStorageDatabaseViews(views []admin20231115002.DataLakeApiBase) []DataLakeViewView { - view := make([]DataLakeViewView, 0) - for ind := range views { - view = append(view, DataLakeViewView{ - Name: views[ind].Name, - Source: views[ind].Source, - Pipeline: views[ind].Pipeline, - }) - } - return view -} - -func flattenDataLakeStorageStores(stores []admin20231115002.DataLakeStoreSettings) []StoreDetail { - store := make([]StoreDetail, 0) - for ind := range stores { - store = append(store, StoreDetail{ - Name: stores[ind].Name, - Provider: &stores[ind].Provider, - Region: stores[ind].Region, - Bucket: stores[ind].Bucket, - Prefix: stores[ind].Prefix, - Delimiter: stores[ind].Delimiter, - IncludeTags: stores[ind].IncludeTags, - AdditionalStorageClasses: stores[ind].AdditionalStorageClasses, - }) - } - return store -} - -func expandDataLakeAwsBlock(cloudProviderConfig DataLakeCloudProviderConfigView) admin20231115002.DataLakeAWSCloudProviderConfig { - awsConfig := admin20231115002.DataLakeAWSCloudProviderConfig{} - if cloudProviderConfig.Aws != nil { - awsConfig.ExternalId = cloudProviderConfig.Aws.ExternalId - awsConfig.IamAssumedRoleARN = cloudProviderConfig.Aws.IamAssumedRoleARN - awsConfig.IamUserARN = cloudProviderConfig.Aws.IamUserARN - awsConfig.RoleId = cast.ToString(cloudProviderConfig.Aws.RoleId) - awsConfig.TestS3Bucket = cast.ToString(cloudProviderConfig.Aws.TestS3Bucket) - } - return awsConfig -} -func expandCloudProviderConfig(currentModel *Model) *admin20231115002.DataLakeCloudProviderConfig { - if currentModel.CloudProviderConfig != nil { - return &admin20231115002.DataLakeCloudProviderConfig{ - Aws: expandDataLakeAwsBlock(*currentModel.CloudProviderConfig), - } - } - return nil -} - -func expandDataLakeDataProcessRegion(dataProcessRegion *DataLakeDataProcessRegionView) *admin20231115002.DataLakeDataProcessRegion { - if dataProcessRegion != nil && dataProcessRegion.Region != nil { - return &admin20231115002.DataLakeDataProcessRegion{ - CloudProvider: cast.ToString(dataProcessRegion.CloudProvider), - Region: cast.ToString(dataProcessRegion.Region), - } - } - return nil -} - -// logger setup function -func setup() { - util.SetupLogger("mongodb-atlas-data-lakes") -} - -// function to validate inputs to all actions -func validateRequest(fields []string, model *Model) *handler.ProgressEvent { - return validator.ValidateModel(fields, model) -} - -// function to track snapshot creation status -func validateProgress(client *util.MongoDBClient, currentModel *Model, targetState string) (handler.ProgressEvent, error) { - projectID := *currentModel.ProjectId - tenantName := *currentModel.TenantName - isReady, state, err := dataLakeIsReady(client, projectID, tenantName, targetState) - if err != nil { - return handler.ProgressEvent{}, err - } - - if !isReady { - p := handler.NewProgressEvent() - p.ResourceModel = currentModel - p.OperationStatus = handler.InProgress - p.CallbackDelaySeconds = 35 - p.Message = "Pending" - p.CallbackContext = map[string]interface{}{ - "status": state, - "project_id": *currentModel.ProjectId, - } - return p, nil - } - - p := handler.NewProgressEvent() - p.ResourceModel = currentModel - p.OperationStatus = handler.Success - p.Message = "Complete" - return p, nil -} - -func isExist(currentModel *Model, client *util.MongoDBClient) bool { - projectID := *currentModel.ProjectId - tenantName := *currentModel.TenantName - dataLake, _, err := client.Atlas20231115002.DataFederationApi.GetFederatedDatabase(context.Background(), projectID, tenantName).Execute() - return err == nil && dataLake != nil -} - -// function to check if snapshot already exist in atlas -func dataLakeIsReady(client *util.MongoDBClient, projectID, name, targetState string) (isReady bool, status string, err error) { - dataLake, _, err := client.Atlas20231115002.DataFederationApi.GetFederatedDatabase(context.Background(), projectID, name).Execute() - if err != nil { - return false, "", err - } - - if dataLake != nil { - return *dataLake.State == targetState, *dataLake.State, nil - } - - return false, "", nil -} - -func convertToModel(dataLake *admin20231115002.DataLakeTenant, currentModel *Model) *Model { - var result = new(Model) - - result.Profile = currentModel.Profile // cfn test - result.TenantName = dataLake.Name - result.State = dataLake.State - result.ProjectId = dataLake.GroupId - result.CloudProviderConfig = &DataLakeCloudProviderConfigView{ - Aws: flattenAWSBlock(dataLake.CloudProviderConfig), - } - result.DataProcessRegion = flattenDataLakeProcessRegion(dataLake.DataProcessRegion) - result.Storage = &DataLakeStorageView{ - Databases: flattenDataLakeStorageDatabases(dataLake.Storage.Databases), - Stores: flattenDataLakeStorageStores(dataLake.Storage.Stores), - } - result.Hostnames = dataLake.Hostnames - result.State = dataLake.State - - return result -} diff --git a/cfn-resources/datalakes/docs/README.md b/cfn-resources/datalakes/docs/README.md deleted file mode 100644 index 2d34dc07b..000000000 --- a/cfn-resources/datalakes/docs/README.md +++ /dev/null @@ -1,151 +0,0 @@ -# MongoDB::Atlas::DataLakes - -Returns, adds, edits, and removes Federated Database Instances. - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "Type" : "MongoDB::Atlas::DataLakes",
-    "Properties" : {
-        "CloudProviderConfig" : DataLakeCloudProviderConfigView,
-        "DataProcessRegion" : DataLakeDataProcessRegionView,
-        "EndDate" : Double,
-        "ProjectId" : String,
-        "Profile" : String,
-        "SkipRoleValidation" : Boolean,
-        "Storage" : DataLakeStorageView,
-        "TenantName" : String,
-    }
-}
-
- -### YAML - -
-Type: MongoDB::Atlas::DataLakes
-Properties:
-    CloudProviderConfig: DataLakeCloudProviderConfigView
-    DataProcessRegion: DataLakeDataProcessRegionView
-    EndDate: Double
-    ProjectId: String
-    Profile: String
-    SkipRoleValidation: Boolean
-    Storage: DataLakeStorageView
-    TenantName: String
-
- -## Properties - -#### CloudProviderConfig - -_Required_: No - -_Type_: DataLakeCloudProviderConfigView - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### DataProcessRegion - -_Required_: No - -_Type_: DataLakeDataProcessRegionView - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### EndDate - -Timestamp that specifies the end point for the range of log messages to download. MongoDB Cloud expresses this timestamp in the number of seconds that have elapsed since the UNIX epoch. - -_Required_: No - -_Type_: Double - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### ProjectId - -Unique 24-hexadecimal digit string that identifies your project. - -_Required_: No - -_Type_: String - -_Minimum_: 24 - -_Maximum_: 24 - -_Pattern_: ^([a-f0-9]{24})$ - -_Update requires_: [Replacement](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-replacement) - -#### Profile - -The profile is defined in AWS Secret manager. See [Secret Manager Profile setup](../../../examples/profile-secret.yaml). - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### SkipRoleValidation - -Flag that indicates whether this request should check if the requesting IAM role can read from the S3 bucket. AWS checks if the role can list the objects in the bucket before writing to it. Some IAM roles only need write permissions. This flag allows you to skip that check. - -_Required_: No - -_Type_: Boolean - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Storage - -_Required_: No - -_Type_: DataLakeStorageView - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### TenantName - -Human-readable label that identifies the Federated Database to remove. - -_Required_: No - -_Type_: String - -_Update requires_: [Replacement](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-replacement) - -## Return Values - -### Fn::GetAtt - -The `Fn::GetAtt` intrinsic function returns a value for a specified attribute of this type. The following are the available attributes and sample return values. - -For more information about using the `Fn::GetAtt` intrinsic function, see [Fn::GetAtt](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/intrinsic-function-reference-getatt.html). - -#### Storage - -Returns the Storage value. - -#### StartDate - -Timestamp that specifies the starting point for the range of log messages to download. MongoDB Cloud expresses this timestamp in the number of seconds that have elapsed since the UNIX epoch. - -#### Hostnames - -Human-readable label that identifies the Federated Database to update. - -#### State - -Human-readable label that identifies the Federated Database to update. - -#### TestS3Bucket - -Returns the TestS3Bucket value. - diff --git a/cfn-resources/datalakes/docs/datalakeawscloudproviderconfigview.md b/cfn-resources/datalakes/docs/datalakeawscloudproviderconfigview.md deleted file mode 100644 index 081b3fb89..000000000 --- a/cfn-resources/datalakes/docs/datalakeawscloudproviderconfigview.md +++ /dev/null @@ -1,84 +0,0 @@ -# MongoDB::Atlas::DataLakes DataLakeAWSCloudProviderConfigView - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "ExternalId" : String,
-    "IamAssumedRoleARN" : String,
-    "IamUserARN" : String,
-    "RoleId" : String,
-    "TestS3Bucket" : String
-}
-
- -### YAML - -
-ExternalId: String
-IamAssumedRoleARN: String
-IamUserARN: String
-RoleId: String
-TestS3Bucket: String
-
- -## Properties - -#### ExternalId - -Unique identifier associated with the Identity and Access Management (IAM) role that the data lake assumes when accessing the data stores. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### IamAssumedRoleARN - -Amazon Resource Name (ARN) of the Identity and Access Management (IAM) role that the data lake assumes when accessing data stores. - -_Required_: No - -_Type_: String - -_Minimum_: 20 - -_Maximum_: 2048 - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### IamUserARN - -Amazon Resource Name (ARN) of the user that the data lake assumes when accessing data stores. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### RoleId - -Unique identifier of the role that the data lake can use to access the data stores.Required if specifying cloudProviderConfig. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### TestS3Bucket - -Name of the S3 data bucket that the provided role ID is authorized to access.Required if specifying cloudProviderConfig. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/datalakes/docs/datalakecloudproviderconfigview.md b/cfn-resources/datalakes/docs/datalakecloudproviderconfigview.md deleted file mode 100644 index 9f4d6b0ce..000000000 --- a/cfn-resources/datalakes/docs/datalakecloudproviderconfigview.md +++ /dev/null @@ -1,30 +0,0 @@ -# MongoDB::Atlas::DataLakes DataLakeCloudProviderConfigView - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "Aws" : DataLakeAWSCloudProviderConfigView
-}
-
- -### YAML - -
-Aws: DataLakeAWSCloudProviderConfigView
-
- -## Properties - -#### Aws - -_Required_: No - -_Type_: DataLakeAWSCloudProviderConfigView - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/datalakes/docs/datalakedatabasecollectionview.md b/cfn-resources/datalakes/docs/datalakedatabasecollectionview.md deleted file mode 100644 index f2598ff49..000000000 --- a/cfn-resources/datalakes/docs/datalakedatabasecollectionview.md +++ /dev/null @@ -1,45 +0,0 @@ -# MongoDB::Atlas::DataLakes DataLakeDatabaseCollectionView - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "DataSources" : [ DataLakeDatabaseDataSourceView, ... ],
-    "Name" : String
-}
-
- -### YAML - -
-DataSources: 
-      - DataLakeDatabaseDataSourceView
-Name: String
-
- -## Properties - -#### DataSources - -Array that contains the data stores that map to a collection for this data lake. - -_Required_: No - -_Type_: List of DataLakeDatabaseDataSourceView - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Name - -Human-readable label that identifies the collection to which MongoDB Cloud maps the data in the data stores. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/datalakes/docs/datalakedatabasedatasourceview.md b/cfn-resources/datalakes/docs/datalakedatabasedatasourceview.md deleted file mode 100644 index 5a37daed8..000000000 --- a/cfn-resources/datalakes/docs/datalakedatabasedatasourceview.md +++ /dev/null @@ -1,119 +0,0 @@ -# MongoDB::Atlas::DataLakes DataLakeDatabaseDataSourceView - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "AllowInsecure" : Boolean,
-    "Collection" : String,
-    "CollectionRegex" : String,
-    "Database" : String,
-    "DefaultFormat" : String,
-    "Path" : String,
-    "StoreName" : String,
-    "Urls" : [ String, ... ]
-}
-
- -### YAML - -
-AllowInsecure: Boolean
-Collection: String
-CollectionRegex: String
-Database: String
-DefaultFormat: String
-Path: String
-StoreName: String
-Urls: 
-      - String
-
- -## Properties - -#### AllowInsecure - -Flag that validates the scheme in the specified URLs. If `true`, allows insecure `HTTP` scheme, doesn't verify the server's certificate chain and hostname, and accepts any certificate with any hostname presented by the server. If `false`, allows secure `HTTPS` scheme only. - -_Required_: No - -_Type_: Boolean - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Collection - -Human-readable label that identifies the collection in the database. For creating a wildcard (`*`) collection, you must omit this parameter. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### CollectionRegex - -Regex pattern to use for creating the wildcard (*) collection. To learn more about the regex syntax, see [Go programming language](https://pkg.go.dev/regexp). - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Database - -Human-readable label that identifies the database, which contains the collection in the cluster. You must omit this parameter to generate wildcard (`*`) collections for dynamically generated databases. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### DefaultFormat - -File format that MongoDB Cloud uses if it encounters a file without a file extension while searching **storeName**. - -_Required_: No - -_Type_: String - -_Allowed Values_: .avro | .avro.gz | .bson | .bson.gz | .csv | .json | .json.gz | .orc | .tsv | .tsv.gz - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Path - -File path that controls how MongoDB Cloud searches for and parses files in the **storeName** before mapping them to a collection.Specify ``/`` to capture all files and folders from the ``prefix`` path. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### StoreName - -Human-readable label that identifies the data store that MongoDB Cloud maps to the collection. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Urls - -URLs of the publicly accessible data files. You can't specify URLs that require authentication. Atlas Data Lake creates a partition for each URL. If empty or omitted, Data Lake uses the URLs from the store specified in the **dataSources.storeName** parameter. - -_Required_: No - -_Type_: List of String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/datalakes/docs/datalakedatabaseview.md b/cfn-resources/datalakes/docs/datalakedatabaseview.md deleted file mode 100644 index 00ef6e7e5..000000000 --- a/cfn-resources/datalakes/docs/datalakedatabaseview.md +++ /dev/null @@ -1,70 +0,0 @@ -# MongoDB::Atlas::DataLakes DataLakeDatabaseView - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "Collections" : [ DataLakeDatabaseCollectionView, ... ],
-    "MaxWildcardCollections" : Integer,
-    "Name" : String,
-    "Views" : [ DataLakeViewView, ... ]
-}
-
- -### YAML - -
-Collections: 
-      - DataLakeDatabaseCollectionView
-MaxWildcardCollections: Integer
-Name: String
-Views: 
-      - DataLakeViewView
-
- -## Properties - -#### Collections - -Array of collections and data sources that map to a ``stores`` data store. - -_Required_: No - -_Type_: List of DataLakeDatabaseCollectionView - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### MaxWildcardCollections - -Maximum number of wildcard collections in the database. This only applies to S3 data sources. - -_Required_: No - -_Type_: Integer - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Name - -Human-readable label that identifies the database to which the data lake maps data. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Views - -Array of aggregation pipelines that apply to the collection. This only applies to S3 data sources. - -_Required_: No - -_Type_: List of DataLakeViewView - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/datalakes/docs/datalakedataprocessregionview.md b/cfn-resources/datalakes/docs/datalakedataprocessregionview.md deleted file mode 100644 index 2ca112a90..000000000 --- a/cfn-resources/datalakes/docs/datalakedataprocessregionview.md +++ /dev/null @@ -1,48 +0,0 @@ -# MongoDB::Atlas::DataLakes DataLakeDataProcessRegionView - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "CloudProvider" : String,
-    "Region" : String
-}
-
- -### YAML - -
-CloudProvider: String
-Region: String
-
- -## Properties - -#### CloudProvider - -Name of the cloud service that hosts the data lake's data stores. - -_Required_: No - -_Type_: String - -_Allowed Values_: AWS | GCP | AZURE | TENANT | SERVERLESS - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Region - -Name of the region to which the data lake routes client connections. - -_Required_: No - -_Type_: String - -_Allowed Values_: DUBLIN_IRL | FRANKFURT_DEU | LONDON_GBR | MUMBAI_IND | OREGON_USA | SYDNEY_AUS | VIRGINIA_USA - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/datalakes/docs/datalakestorageview.md b/cfn-resources/datalakes/docs/datalakestorageview.md deleted file mode 100644 index ea3a54e0a..000000000 --- a/cfn-resources/datalakes/docs/datalakestorageview.md +++ /dev/null @@ -1,46 +0,0 @@ -# MongoDB::Atlas::DataLakes DataLakeStorageView - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "Databases" : [ DataLakeDatabaseView, ... ],
-    "Stores" : [ StoreDetail, ... ]
-}
-
- -### YAML - -
-Databases: 
-      - DataLakeDatabaseView
-Stores: 
-      - StoreDetail
-
- -## Properties - -#### Databases - -Array that contains the queryable databases and collections for this data lake. - -_Required_: No - -_Type_: List of DataLakeDatabaseView - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Stores - -Array that contains the data stores for the data lake. - -_Required_: No - -_Type_: List of StoreDetail - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/datalakes/docs/datalakeviewview.md b/cfn-resources/datalakes/docs/datalakeviewview.md deleted file mode 100644 index 35e17759e..000000000 --- a/cfn-resources/datalakes/docs/datalakeviewview.md +++ /dev/null @@ -1,56 +0,0 @@ -# MongoDB::Atlas::DataLakes DataLakeViewView - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "Name" : String,
-    "Pipeline" : String,
-    "Source" : String
-}
-
- -### YAML - -
-Name: String
-Pipeline: String
-Source: String
-
- -## Properties - -#### Name - -Human-readable label that identifies the view, which corresponds to an aggregation pipeline on a collection. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Pipeline - -Aggregation pipeline stages to apply to the source collection. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Source - -Human-readable label that identifies the source collection for the view. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/datalakes/docs/storedetail.md b/cfn-resources/datalakes/docs/storedetail.md deleted file mode 100644 index 8ceb2ab68..000000000 --- a/cfn-resources/datalakes/docs/storedetail.md +++ /dev/null @@ -1,119 +0,0 @@ -# MongoDB::Atlas::DataLakes StoreDetail - -Configuration information for each data store and its mapping to MongoDB Cloud databases. - -## Syntax - -To declare this entity in your AWS CloudFormation template, use the following syntax: - -### JSON - -
-{
-    "Name" : String,
-    "Provider" : String,
-    "Region" : String,
-    "Bucket" : String,
-    "Prefix" : String,
-    "Delimiter" : String,
-    "IncludeTags" : Boolean,
-    "AdditionalStorageClasses" : [ String, ... ]
-}
-
- -### YAML - -
-Name: String
-Provider: String
-Region: String
-Bucket: String
-Prefix: String
-Delimiter: String
-IncludeTags: Boolean
-AdditionalStorageClasses: 
-      - String
-
- -## Properties - -#### Name - -Human-readable label that identifies the data store. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Provider - -Human-readable label that identifies the Federated Database to update. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Region - -Human-readable label that identifies the Federated Database to update. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Bucket - -Human-readable label that identifies the Federated Database to update. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Prefix - -Human-readable label that identifies the Federated Database to update. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### Delimiter - -Human-readable label that identifies the Federated Database to update. - -_Required_: No - -_Type_: String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### IncludeTags - -Human-readable label that identifies the Federated Database to update. - -_Required_: No - -_Type_: Boolean - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - -#### AdditionalStorageClasses - -Human-readable label that identifies the Federated Database to update. - -_Required_: No - -_Type_: List of String - -_Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt) - diff --git a/cfn-resources/datalakes/mongodb-atlas-datalakes.json b/cfn-resources/datalakes/mongodb-atlas-datalakes.json deleted file mode 100644 index b187abaf6..000000000 --- a/cfn-resources/datalakes/mongodb-atlas-datalakes.json +++ /dev/null @@ -1,582 +0,0 @@ -{ - "additionalProperties": false, - "definitions": { - "DataLakeAWSCloudProviderConfigView": { - "type": "object", - "properties": { - "ExternalId": { - "type": "string", - "description": "Unique identifier associated with the Identity and Access Management (IAM) role that the data lake assumes when accessing the data stores." - }, - "IamAssumedRoleARN": { - "type": "string", - "description": "Amazon Resource Name (ARN) of the Identity and Access Management (IAM) role that the data lake assumes when accessing data stores.", - "maxLength": 2048, - "minLength": 20 - }, - "IamUserARN": { - "type": "string", - "description": "Amazon Resource Name (ARN) of the user that the data lake assumes when accessing data stores." - }, - "RoleId": { - "type": "string", - "description": "Unique identifier of the role that the data lake can use to access the data stores.Required if specifying cloudProviderConfig." - }, - "TestS3Bucket": { - "type": "string", - "description": "Name of the S3 data bucket that the provided role ID is authorized to access.Required if specifying cloudProviderConfig." - } - }, - "additionalProperties": false - }, - "DataLakeCloudProviderConfigView": { - "type": "object", - "properties": { - "Aws": { - "type": "object", - "description": "Name of the cloud service that hosts the data lake's data stores.", - "$ref": "#/definitions/DataLakeAWSCloudProviderConfigView" - } - }, - "additionalProperties": false - }, - "DataLakeDataProcessRegionView": { - "type": "object", - "properties": { - "CloudProvider": { - "type": "string", - "description": "Name of the cloud service that hosts the data lake's data stores.", - "enum": [ - "AWS", - "GCP", - "AZURE", - "TENANT", - "SERVERLESS" - ] - }, - "Region": { - "type": "string", - "description": "Name of the region to which the data lake routes client connections.", - "enum": [ - "DUBLIN_IRL", - "FRANKFURT_DEU", - "LONDON_GBR", - "MUMBAI_IND", - "OREGON_USA", - "SYDNEY_AUS", - "VIRGINIA_USA" - ] - } - }, - "additionalProperties": false - }, - "DataLakeDatabaseCollectionView": { - "type": "object", - "properties": { - "DataSources": { - "type": "array", - "insertionOrder": false, - "description": "Array that contains the data stores that map to a collection for this data lake.", - "items": { - "$ref": "#/definitions/DataLakeDatabaseDataSourceView", - "type": "object" - } - }, - "Name": { - "type": "string", - "description": "Human-readable label that identifies the collection to which MongoDB Cloud maps the data in the data stores." - } - }, - "additionalProperties": false - }, - "DataLakeDatabaseDataSourceView": { - "type": "object", - "properties": { - "AllowInsecure": { - "type": "boolean", - "description": "Flag that validates the scheme in the specified URLs. If `true`, allows insecure `HTTP` scheme, doesn't verify the server's certificate chain and hostname, and accepts any certificate with any hostname presented by the server. If `false`, allows secure `HTTPS` scheme only." - }, - "Collection": { - "type": "string", - "description": "Human-readable label that identifies the collection in the database. For creating a wildcard (`*`) collection, you must omit this parameter." - }, - "CollectionRegex": { - "type": "string", - "description": "Regex pattern to use for creating the wildcard (*) collection. To learn more about the regex syntax, see [Go programming language](https://pkg.go.dev/regexp)." - }, - "Database": { - "type": "string", - "description": "Human-readable label that identifies the database, which contains the collection in the cluster. You must omit this parameter to generate wildcard (`*`) collections for dynamically generated databases." - }, - "DefaultFormat": { - "type": "string", - "description": "File format that MongoDB Cloud uses if it encounters a file without a file extension while searching **storeName**.", - "enum": [ - ".avro", - ".avro.gz", - ".bson", - ".bson.gz", - ".csv", - ".json", - ".json.gz", - ".orc", - ".tsv", - ".tsv.gz" - ] - }, - "Path": { - "type": "string", - "description": "File path that controls how MongoDB Cloud searches for and parses files in the **storeName** before mapping them to a collection.Specify ``/`` to capture all files and folders from the ``prefix`` path." - }, - "StoreName": { - "type": "string", - "description": "Human-readable label that identifies the data store that MongoDB Cloud maps to the collection." - }, - "Urls": { - "type": "array", - "insertionOrder": false, - "description": "URLs of the publicly accessible data files. You can't specify URLs that require authentication. Atlas Data Lake creates a partition for each URL. If empty or omitted, Data Lake uses the URLs from the store specified in the **dataSources.storeName** parameter.", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false - }, - "DataLakeDatabaseView": { - "type": "object", - "properties": { - "Collections": { - "type": "array", - "insertionOrder": false, - "description": "Array of collections and data sources that map to a ``stores`` data store.", - "items": { - "$ref": "#/definitions/DataLakeDatabaseCollectionView", - "type": "object" - } - }, - "MaxWildcardCollections": { - "type": "integer", - "description": "Maximum number of wildcard collections in the database. This only applies to S3 data sources." - }, - "Name": { - "type": "string", - "description": "Human-readable label that identifies the database to which the data lake maps data." - }, - "Views": { - "type": "array", - "insertionOrder": false, - "description": "Array of aggregation pipelines that apply to the collection. This only applies to S3 data sources.", - "items": { - "$ref": "#/definitions/DataLakeViewView", - "type": "object" - } - } - }, - "additionalProperties": false - }, - "DataLakeStorageView": { - "type": "object", - "properties": { - "Databases": { - "type": "array", - "insertionOrder": false, - "description": "Array that contains the queryable databases and collections for this data lake.", - "items": { - "$ref": "#/definitions/DataLakeDatabaseView", - "type": "object" - } - }, - "Stores": { - "type": "array", - "insertionOrder": false, - "description": "Array that contains the data stores for the data lake.", - "items": { - "$ref": "#/definitions/StoreDetail", - "type": "object" - } - } - }, - "additionalProperties": false - }, - "StoreDetail": { - "type": "object", - "description": "Configuration information for each data store and its mapping to MongoDB Cloud databases.", - "properties": { - "Name": { - "description": "Human-readable label that identifies the data store.", - "type": "string" - }, - "Provider": { - "description": "Human-readable label that identifies the Federated Database to update.", - "type": "string" - }, - "Region": { - "description": "Human-readable label that identifies the Federated Database to update.", - "type": "string" - }, - "Bucket": { - "description": "Human-readable label that identifies the Federated Database to update.", - "type": "string" - }, - "Prefix": { - "description": "Human-readable label that identifies the Federated Database to update.", - "type": "string" - }, - "Delimiter": { - "description": "Human-readable label that identifies the Federated Database to update.", - "type": "string" - }, - "IncludeTags": { - "description": "Human-readable label that identifies the Federated Database to update.", - "type": "boolean" - }, - "AdditionalStorageClasses": { - "type": "array", - "insertionOrder": false, - "description": "Human-readable label that identifies the Federated Database to update.", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false - }, - "DataLakeViewView": { - "type": "object", - "properties": { - "Name": { - "type": "string", - "description": "Human-readable label that identifies the view, which corresponds to an aggregation pipeline on a collection." - }, - "Pipeline": { - "type": "string", - "description": "Aggregation pipeline stages to apply to the source collection." - }, - "Source": { - "type": "string", - "description": "Human-readable label that identifies the source collection for the view." - } - }, - "additionalProperties": false - }, - "NDSDataLakeAWSCloudProviderConfigView": { - "type": "object", - "properties": { - "RoleId": { - "type": "string", - "description": "Unique identifier of the role that the data lake can use to access the data stores.Required if specifying cloudProviderConfig." - }, - "TestS3Bucket": { - "type": "string", - "description": "Name of the S3 data bucket that the provided role ID is authorized to access.Required if specifying cloudProviderConfig." - } - }, - "additionalProperties": false - }, - "NDSDataLakeCloudProviderConfigView": { - "type": "object", - "properties": { - "Aws": { - "type": "object", - "description": "Name of the cloud service that hosts the data lake's data stores.", - "$ref": "#/definitions/NDSDataLakeAWSCloudProviderConfigView" - } - }, - "additionalProperties": false - }, - "NDSDataLakeDataProcessRegionView": { - "type": "object", - "properties": { - "CloudProvider": { - "type": "string", - "description": "Name of the cloud service that hosts the data lake's data stores.", - "enum": [ - "NONE", - "AWS", - "GCP", - "AZURE", - "FREE", - "SERVERLESS" - ] - }, - "Region": { - "type": "string", - "description": "Name of the region to which the data lake routes connections for data processing.", - "enum": [ - "DUBLIN_IRL", - "FRANKFURT_DEU", - "LONDON_GBR", - "MUMBAI_IND", - "OREGON_USA", - "SYDNEY_AUS", - "VIRGINIA_USA" - ] - } - }, - "additionalProperties": false - }, - "NDSDataLakeDatabaseCollectionView": { - "type": "object", - "properties": { - "DataSources": { - "type": "array", - "insertionOrder": false, - "description": "Array that contains the data stores that map to a collection for this data lake.", - "items": { - "$ref": "#/definitions/NDSDataLakeDatabaseDataSourceView", - "type": "object" - } - }, - "Name": { - "type": "string", - "description": "Human-readable label that identifies the collection to which MongoDB Cloud maps the data in the data stores." - } - }, - "additionalProperties": false - }, - "NDSDataLakeDatabaseDataSourceView": { - "type": "object", - "properties": { - "AllowInsecure": { - "type": "boolean", - "description": "Flag that validates the scheme in the specified URLs. If `true`, allows insecure `HTTP` scheme, doesn't verify the server's certificate chain and hostname, and accepts any certificate with any hostname presented by the server. If `false`, allows secure `HTTPS` scheme only." - }, - "Collection": { - "type": "string", - "description": "Human-readable label that identifies the collection in the database. For creating a wildcard (`*`) collection, you must omit this parameter." - }, - "CollectionRegex": { - "type": "string", - "description": "Regex pattern to use for creating the wildcard (*) collection. To learn more about the regex syntax, see [Go programming language](https://pkg.go.dev/regexp)." - }, - "Database": { - "type": "string", - "description": "Human-readable label that identifies the database, which contains the collection in the cluster. You must omit this parameter to generate wildcard (`*`) collections for dynamically generated databases." - }, - "DefaultFormat": { - "type": "string", - "description": "File format that MongoDB Cloud uses if it encounters a file without a file extension while searching **storeName**.", - "enum": [ - ".avro", - ".avro.gz", - ".bson", - ".bson.gz", - ".csv", - ".json", - ".json.gz", - ".orc", - ".tsv", - ".tsv.gz" - ] - }, - "Path": { - "type": "string", - "description": "File path that controls how MongoDB Cloud searches for and parses files in the **storeName** before mapping them to a collection.Specify ``/`` to capture all files and folders from the ``prefix`` path." - }, - "StoreName": { - "type": "string", - "description": "Human-readable label that identifies the data store to map to the **collection**. Must match the **name** of an object in the **stores** array." - }, - "Urls": { - "type": "array", - "insertionOrder": false, - "description": "URLs of the publicly accessible data files. You can't specify URLs that require authentication. Atlas Data Lake creates a partition for each URL. If empty or omitted, Data Lake uses the URLs from the store specified in the **dataSources.storeName** parameter.", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false - }, - "NDSDataLakeDatabaseView": { - "type": "object", - "properties": { - "Collections": { - "type": "array", - "insertionOrder": false, - "description": "Array of collections and data sources that map to a ``stores`` data store.", - "items": { - "$ref": "#/definitions/NDSDataLakeDatabaseCollectionView", - "type": "object" - } - }, - "MaxWildcardCollections": { - "type": "integer", - "description": "Maximum number of wildcard collections in the database." - }, - "Name": { - "type": "string", - "description": "Human-readable label that identifies the database to which the data lake maps data." - }, - "Views": { - "type": "array", - "insertionOrder": false, - "description": "Array of aggregation pipelines that apply to the collection.", - "items": { - "$ref": "#/definitions/NDSDataLakeViewView", - "type": "object" - } - } - }, - "additionalProperties": false - }, - "NDSDataLakeStorageView": { - "type": "object", - "properties": { - "Databases": { - "type": "array", - "insertionOrder": false, - "description": "Array that contains the queryable databases and collections for this data lake.", - "items": { - "$ref": "#/definitions/NDSDataLakeDatabaseView", - "type": "object" - } - }, - "Stores": { - "type": "array", - "insertionOrder": false, - "description": "Array that contains the data stores for the data lake.", - "items": { - "$ref": "#/definitions/StoreDetail", - "type": "object" - } - } - }, - "additionalProperties": false - }, - "NDSDataLakeViewView": { - "type": "object", - "properties": { - "Name": { - "type": "string", - "description": "Human-readable label that identifies the view, which corresponds to an aggregation pipeline on a collection." - }, - "Pipeline": { - "type": "string", - "description": "Aggregation pipeline stages to apply to the source collection." - }, - "Source": { - "type": "string", - "description": "Human-readable label that identifies the source collection for the view." - } - }, - "additionalProperties": false - }, - "Provider": { - "type": "object", - "properties": { - "Name": { - "type": "string", - "description": "Human-readable label that identifies the data store. The **databases.[n].collections.[n].dataSources.[n].storeName** field references this values as part of the mapping configuration. To use MongoDB Cloud as a data store, the data lake requires a serverless instance or an `M10` or higher cluster." - }, - "Provider": { - "type": "string" - } - }, - "additionalProperties": false - } - }, - "description": "Returns, adds, edits, and removes Federated Database Instances.", - "handlers": { - "create": { - "permissions": [ - "secretsmanager:GetSecretValue" - ] - }, - "read": { - "permissions": [ - "secretsmanager:GetSecretValue" - ] - }, - "update": { - "permissions": [ - "secretsmanager:GetSecretValue" - ] - }, - "delete": { - "permissions": [ - "secretsmanager:GetSecretValue" - ] - } - }, - "properties": { - "CloudProviderConfig": { - "type": "object", - "description": "Cloud provider linked to this data lake.", - "$ref": "#/definitions/DataLakeCloudProviderConfigView" - }, - "DataProcessRegion": { - "type": "object", - "description": "Information about the cloud provider region to which the data lake routes client connections. MongoDB Cloud supports AWS only.", - "$ref": "#/definitions/DataLakeDataProcessRegionView" - }, - "EndDate": { - "type": "number", - "description": "Timestamp that specifies the end point for the range of log messages to download. MongoDB Cloud expresses this timestamp in the number of seconds that have elapsed since the UNIX epoch." - }, - "ProjectId": { - "type": "string", - "description": "Unique 24-hexadecimal digit string that identifies your project.", - "maxLength": 24, - "minLength": 24, - "pattern": "^([a-f0-9]{24})$" - }, - "Profile": { - "type": "string", - "description": "The profile is defined in AWS Secret manager. See [Secret Manager Profile setup](../../../examples/profile-secret.yaml).", - "default": "default" - }, - "SkipRoleValidation": { - "type": "boolean", - "description": "Flag that indicates whether this request should check if the requesting IAM role can read from the S3 bucket. AWS checks if the role can list the objects in the bucket before writing to it. Some IAM roles only need write permissions. This flag allows you to skip that check." - }, - "StartDate": { - "type": "number", - "description": "Timestamp that specifies the starting point for the range of log messages to download. MongoDB Cloud expresses this timestamp in the number of seconds that have elapsed since the UNIX epoch." - }, - "Storage": { - "type": "object", - "description": "Configuration information for each data store and its mapping to MongoDB Cloud databases.", - "$ref": "#/definitions/DataLakeStorageView" - }, - "TenantName": { - "type": "string", - "description": "Human-readable label that identifies the Federated Database to remove." - }, - "Hostnames": { - "type": "array", - "insertionOrder": false, - "description": "Human-readable label that identifies the Federated Database to update.", - "items": { - "type": "string" - } - }, - "State": { - "type": "string", - "description": "Human-readable label that identifies the Federated Database to update." - } - }, - "primaryIdentifier": [ - "/properties/ProjectId", - "/properties/TenantName", - "/properties/Profile" - ], - "createOnlyProperties": [ - "/properties/ProjectId", - "/properties/TenantName" - ], - "readOnlyProperties": [ - "/properties/Storage", - "/properties/StartDate", - "/properties/Hostnames", - "/properties/State", - "/properties/CloudProviderConfig/Aws/TestS3Bucket" - ], - "typeName": "MongoDB::Atlas::DataLakes", - "sourceUrl": "https://github.com/mongodb/mongodbatlas-cloudformation-resources/tree/master/cfn-resources/datalakes", - "documentationUrl": "https://github.com/mongodb/mongodbatlas-cloudformation-resources/blob/master/cfn-resources/datalakes/README.md", - "tagging": { - "taggable": false - } -} diff --git a/cfn-resources/datalakes/resource-role.yaml b/cfn-resources/datalakes/resource-role.yaml deleted file mode 100644 index 4ac0bfcdd..000000000 --- a/cfn-resources/datalakes/resource-role.yaml +++ /dev/null @@ -1,47 +0,0 @@ -AWSTemplateFormatVersion: "2010-09-09" -Description: > - This CloudFormation template creates a role assumed by CloudFormation - during CRUDL operations to mutate resources on behalf of the customer. - -Resources: - ExecutionRole: - Type: AWS::IAM::Role - Properties: - MaxSessionDuration: 8400 - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: resources.cloudformation.amazonaws.com - Action: sts:AssumeRole - Path: "/" - Policies: - - PolicyName: ResourceTypePolicy - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - "secretsmanager:GetSecretValue" - - "ec2:CreateVpcEndpoint" - - "ec2:DeleteVpcEndpoints" - - "cloudformation:CreateResource" - - "cloudformation:DeleteResource" - - "cloudformation:GetResource" - - "cloudformation:GetResourceRequestStatus" - - "cloudformation:ListResources" - - "cloudformation:UpdateResource" - - "iam:AttachRolePolicy" - - "iam:CreateRole" - - "iam:DeleteRole" - - "iam:GetRole" - - "iam:GetRolePolicy" - - "iam:ListAttachedRolePolicies" - - "iam:ListRolePolicies" - - "iam:PutRolePolicy" - Resource: "*" -Outputs: - ExecutionRoleArn: - Value: - Fn::GetAtt: ExecutionRole.Arn diff --git a/cfn-resources/datalakes/template.yml b/cfn-resources/datalakes/template.yml deleted file mode 100644 index 1189f485b..000000000 --- a/cfn-resources/datalakes/template.yml +++ /dev/null @@ -1,29 +0,0 @@ -AWSTemplateFormatVersion: "2010-09-09" -Transform: AWS::Serverless-2016-10-31 -Description: AWS SAM template for the MongoDB::Atlas::DataLakes resource type - -Globals: - Function: - Timeout: 180 # docker start-up times can be long for SAM CLI - MemorySize: 256 - -Resources: - TypeFunction: - Type: AWS::Serverless::Function - Properties: - Handler: bootstrap - Runtime: provided.al2 - CodeUri: bin/ - - TestEntrypoint: - Type: AWS::Serverless::Function - Properties: - Handler: bootstrap - Runtime: provided.al2 - CodeUri: bin/ - Environment: - Variables: - MODE: Test - LOG_LEVEL: debug - MONGODB_ATLAS_BASE_URL: - diff --git a/cfn-resources/datalakes/test/README.md b/cfn-resources/datalakes/test/README.md deleted file mode 100644 index c98cb7e87..000000000 --- a/cfn-resources/datalakes/test/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Datalake - -## Impact -The following components use this resource and are potentially impacted by any changes. They should also be validated to ensure the changes do not cause a regression. - - Datalake L1 CDK constructor - - - -## Prerequisites -### Resources needed to run the manual QA -- Atlas project -- Atlas Cluster -- AWS role -- AWS role policy - - - -All resources are created as part of `cfn-testing-helper.sh` - -## Manual QA -Please, follows the steps in [TESTING.md](../../../TESTING.md). - - -### Success criteria when testing the resource -- You should see your data federation in the "Data Federation" page: -![image](https://user-images.githubusercontent.com/5663078/227923171-b9aa0067-d8a0-41b3-94c5-9e5ce5ea222c.png) -## Important Links -- [API Documentation](https://docs-atlas-staging.mongodb.com/cloud-docs/docsworker-xlarge/openapi-docs-test/reference/api-resources-spec/#tag/Data-Federationt) -- [Resource Usage Documentation](https://www.mongodb.com/docs/atlas/data-federation/config/config-adl-datasets/) - -## Unit Testing Locally -The local tests are integrated with the AWS `sam local` and `cfn invoke` tooling features: - -``` -sam local start-lambda --skip-pull-image -``` -then in another shell: -```bash -repo_root=$(git rev-parse --show-toplevel) -source <(${repo_root}/quickstart-mongodb-atlas/scripts/export-mongocli-config.py) -cd ${repo_root}/cfn-resources/datalakes -./test/datalakes.create-sample-cfn-request.sh YourProjectID ExternalId IamAssumedRoleARN IamUserARN RoleId TestS3BucketId > test.request.json -echo "Sample request:" -cat test.request.json -cfn invoke CREATE test.request.json -cfn invoke DELETE test.request.json -``` - -Both CREATE & DELETE tests must pass. \ No newline at end of file diff --git a/cfn-resources/datalakes/test/add-policy.json b/cfn-resources/datalakes/test/add-policy.json deleted file mode 100644 index a19a3a10d..000000000 --- a/cfn-resources/datalakes/test/add-policy.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::536727724300:root" - }, - "Action": "sts:AssumeRole", - "Condition": { - "StringEquals": { - "sts:ExternalId": "0cc1054b-3b73-4a2d-977b-c0ba31157928" - } - } - } - ] -} diff --git a/cfn-resources/datalakes/test/cfn-test-create-inputs.sh b/cfn-resources/datalakes/test/cfn-test-create-inputs.sh deleted file mode 100755 index 29fa9cdd4..000000000 --- a/cfn-resources/datalakes/test/cfn-test-create-inputs.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2023 MongoDB Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# cfn-test-create-inputs.sh -# -# This tool generates json files in the inputs/ for `cfn test`. -# - -set -o nounset -set -o pipefail - -function usage { - echo "usage:$0 " -} - -if [ "$#" -ne 1 ]; then usage; fi -if [[ "$*" == help ]]; then usage; fi - -rm -rf inputs -mkdir inputs - -#project_id - -projectName="${1}" -projectId=$(atlas projects list --output json | jq --arg NAME "${projectName}" -r '.results[] | select(.name==$NAME) | .id') -if [ -z "$projectId" ]; then - projectId=$(atlas projects create "${projectName}" --output=json | jq -r '.id') - - echo -e "Created project \"${projectName}\" with id: ${projectId}\n" -else - echo -e "FOUND project \"${projectName}\" with id: ${projectId}\n" -fi - -clusterName="${projectName}" - -atlas clusters create "${clusterName}" --projectId "${projectId}" --backup --provider AWS --region US_EAST_1 --members 3 --tier M10 --mdbVersion 5.0 --diskSizeGB 10 --output=json -atlas clusters watch "${clusterName}" --projectId "${projectId}" -echo -e "Created Cluster \"${clusterName}\"" - -keyRegion=$AWS_DEFAULT_REGION -if [ -z "$keyRegion" ]; then - keyRegion=$(aws configure get region) -fi - -# shellcheck disable=SC2001 -keyRegionUnderScore=$(echo "$keyRegion" | sed -e "s/-/_/g") -keyRegionUnderScore=$(echo "$keyRegionUnderScore" | tr '[:lower:]' '[:upper:]') -echo "$keyRegion" - -roleName="mongodb-test-datalake-role-${keyRegionUnderScore}" -policyName="atlas-bucket-role-policy-${keyRegionUnderScore}" - -echo "roleName: ${roleName} , policyName: ${policyName}" - -echo -e "--------------------------------create key and key policy document starts ----------------------------\n" - -echo -e "--------------------------------create aws bucket document starts ----------------------------\n" -bucketName="cfntest-demo-test123-${keyRegion}" -aws s3 rb "s3://${bucketName}" --force -aws s3 mb "s3://${bucketName}" --output json - -echo -e "--------------------------------create aws bucket document ends ----------------------------\n" - -roleID=$(atlas cloudProviders accessRoles aws create --projectId "${projectId}" --output json | jq -r '.roleId') -echo -e "--------------------------------Mongo CLI Role creation ends ----------------------------\n" - -echo -e "--------------------------------printing mongodb role details ----------------------------\n" -atlas cloudProviders accessRoles list --projectId "${projectId}" --output json | jq --arg NAME "${projectName}" -r '.awsIamRoles[] |select(.iamAssumedRoleArn |test( "mongodb-test-export-role$")?)' -echo -e "--------------------------------AWS Role policy creation starts ----------------------------\n" - -atlasAWSAccountArn=$(atlas cloudProviders accessRoles list --projectId "${projectId}" --output json | jq --arg roleID "${roleID}" -r '.awsIamRoles[] |select(.roleId |test( $roleID)) |.atlasAWSAccountArn') -atlasAssumedRoleExternalId=$(atlas cloudProviders accessRoles list --projectId "${projectId}" --output json | jq --arg roleID "${roleID}" -r '.awsIamRoles[] |select(.roleId |test( $roleID)) |.atlasAssumedRoleExternalId') -jq --arg atlasAssumedRoleExternalId "$atlasAssumedRoleExternalId" \ - --arg atlasAWSAccountArn "$atlasAWSAccountArn" \ - '.Statement[0].Principal.AWS?|=$atlasAWSAccountArn | .Statement[0].Condition.StringEquals["sts:ExternalId"]?|=$atlasAssumedRoleExternalId' "$(dirname "$0")/role-policy-template.json" >"$(dirname "$0")/add-policy.json" -echo cat add-policy.json -echo -e "--------------------------------AWS Role creation ends ----------------------------\n" - -awsRoleID=$(aws iam get-role --role-name "${roleName}" | jq --arg roleName "${roleName}" -r '.Role | select(.RoleName==$roleName) |.RoleId') -if [ -z "$awsRoleID" ]; then - awsRoleID=$(aws iam create-role --role-name "${roleName}" --assume-role-policy-document "file://$(dirname "$0")/add-policy.json" | jq --arg roleName "${roleName}" -r '.Role | select(.RoleName==$roleName) |.RoleId') - echo -e "No role found, hence creating the role. Created id: ${awsRoleID}\n" -else - aws iam delete-role-policy --role-name "${roleName}" --policy-name "${policyName}" - aws iam delete-role --role-name "${roleName}" - awsRoleID=$(aws iam create-role --role-name "${roleName}" --assume-role-policy-document "file://$(dirname "$0")/add-policy.json" | jq --arg roleName "${roleName}" -r '.Role | select(.RoleName==$roleName) |.RoleId') - echo -e "FOUND id: ${awsRoleID}\n" -fi -echo -e "--------------------------------AWS Role creation ends ----------------------------\n" - -awsArn=$(aws iam get-role --role-name "${roleName}" | jq --arg roleName "${roleName}" -r '.Role | select(.RoleName==$roleName) |.Arn') - -aws iam put-role-policy --role-name "${roleName}" --policy-name "${policyName}" --policy-document "file://$(dirname "$0")/policy.json" -echo -e "--------------------------------attach mongodb Role to AWS Role ends ----------------------------\n" - -echo -e "--------------------------------Role Id : ${roleID} ----------------------------\n" - -# shellcheck disable=SC2001 -awsArne=$(echo "${awsArn}" | sed 's/"//g') - -# TODO Needs change to while loop using get operation -sleep 65 - -atlas cloudProviders accessRoles aws authorize "${roleID}" --projectId "${projectId}" --iamAssumedRoleArn "${awsArne}" -echo -e "--------------------------------authorize mongodb Role ends ----------------------------\n" - -jq --arg org "$MONGODB_ATLAS_ORG_ID" \ - --arg projectId "$projectId" \ - --arg atlasAssumedRoleExternalId "$atlasAssumedRoleExternalId" \ - --arg atlasAWSAccountArn "$atlasAWSAccountArn" \ - --arg AWSAssumedArn "$awsArne" \ - --arg role "$roleID" \ - --arg bucketName "$bucketName" \ - '.TenantName?|=$bucketName |.CloudProviderConfig.Aws.TestS3Bucket?|=$bucketName |.CloudProviderConfig.Aws.RoleId?|=$role |.CloudProviderConfig.Aws.IamUserARN?|=$atlasAWSAccountArn |.CloudProviderConfig.Aws.ExternalId?|=$atlasAssumedRoleExternalId | .CloudProviderConfig.Aws.IamAssumedRoleARN?|=$AWSAssumedArn | .ProjectId?|=$projectId' \ - "$(dirname "$0")/inputs_1_create.template.json" >"inputs/inputs_1_create.json" - -jq --arg org "$MONGODB_ATLAS_ORG_ID" \ - --arg projectId "$projectId" \ - --arg atlasAssumedRoleExternalId "$atlasAssumedRoleExternalId" \ - --arg atlasAWSAccountArn "$atlasAWSAccountArn" \ - --arg AWSAssumedArn "$awsArne" \ - --arg role "$roleID" \ - --arg bucketName "$bucketName" \ - '.TenantName?|=$bucketName |.CloudProviderConfig.Aws.TestS3Bucket?|=$bucketName |.CloudProviderConfig.Aws.RoleId?|=$role |.CloudProviderConfig.Aws.IamUserARN?|=$atlasAWSAccountArn |.CloudProviderConfig.Aws.ExternalId?|=$atlasAssumedRoleExternalId | .CloudProviderConfig.Aws.IamAssumedRoleARN?|=$AWSAssumedArn | .ProjectId?|=$projectId' \ - "$(dirname "$0")/inputs_1_update.template.json" >"inputs/inputs_1_update.json" - -ls -l inputs -echo "mongocli iam projects delete ${projectId} --force" diff --git a/cfn-resources/datalakes/test/cfn-test-delete-inputs.sh b/cfn-resources/datalakes/test/cfn-test-delete-inputs.sh deleted file mode 100755 index 39bd40404..000000000 --- a/cfn-resources/datalakes/test/cfn-test-delete-inputs.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# cfn-test-delete-inputs.sh -# -# This tool deletes the mongodb resources used for `cfn test` as inputs. -# - -set -o errexit -set -o nounset -set -o pipefail - -function usage { - echo "usage:$0 " -} - -project_id=$(jq -r '.ProjectId' ./inputs/inputs_1_create.json) - -# Get a list of clusters for the current project -clusters_response=$(atlas clusters list --projectId "$project_id" 2>&1) - -# Check if the response contains the "GROUP_NOT_FOUND" error message -if [[ "$clusters_response" == *"GROUP_NOT_FOUND"* ]]; then - echo "Project with ID $project_id not found. Continuing with the next project." -fi - -total_count=$(echo "$clusters_response" | jq -r '.totalCount') - -# Check if there are clusters for the project -if [ "$total_count" -eq 0 ]; then - echo "No clusters found for project with ID: $project_id" -else - # Extract cluster names from the response - mapfile -t cluster_names < <(echo "$clusters_response" | jq -r '.results[].name') - - # Delete each cluster within the project - for cluster_name in "${cluster_names[@]}"; do - echo "Deleting cluster with name: $cluster_name" - atlas clusters delete "$cluster_name" --projectId "$project_id" --force - - echo "Waiting for cluster to get deleted" - status=$(atlas clusters describe "${cluster_name}" --projectId "${project_id}" --output=json | jq -r '.stateName') - echo "status: ${status}" - - while [[ "${status}" == "DELETING" ]]; do - sleep 30 - if atlas clusters describe "${cluster_name}" --projectId "${project_id}"; then - status=$(atlas clusters describe "${cluster_name}" --projectId "${project_id}" --output=json | jq -r '.stateName') - else - status="DELETED" - fi - echo "status: ${status}" - done - done -fi - -#delete project -if atlas projects delete "$project_id" --force; then - echo "$project_id project deletion OK" -else - (echo "Failed cleaning project:$project_id" && exit 1) -fi diff --git a/cfn-resources/datalakes/test/datalakes.create-sample-cfn-request.sh b/cfn-resources/datalakes/test/datalakes.create-sample-cfn-request.sh deleted file mode 100644 index 296d25898..000000000 --- a/cfn-resources/datalakes/test/datalakes.create-sample-cfn-request.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# datalakes.create-sample-cfn-request.sh -# -# This tool generates text for a `cfn invoke` request json message. -# - -set -o errexit -set -o nounset -set -o pipefail - -function usage { - echo "usage:$0 " - echo "usage:$0 " - echo "usage:$0 " - echo "usage:$0 " - echo "usage:$0 " - echo "usage:$0 " - exit -} - -if [ "$#" -ne 6 ]; then usage; fi -if [[ "$*" == help ]]; then usage; fi - -projectId="${1}" -atlasAssumedRoleExternalId="${2}" -atlasAWSAccountArn="${3}" -awsArne="${4}" -roleID="${5}" -bucketName="${6}" - -jq --arg org "$MONGODB_ATLAS_ORG_ID" \ - --arg projectId "$projectId" \ - --arg atlasAssumedRoleExternalId "$atlasAssumedRoleExternalId" \ - --arg atlasAWSAccountArn "$atlasAWSAccountArn" \ - --arg AWSAssumedArn "$awsArne" \ - --arg role "$roleID" \ - --arg bucketName "$bucketName" \ - '.desiredResourceState.TenantName?|=$bucketName |.desiredResourceState.CloudProviderConfig.Aws.TestS3Bucket?|=$bucketName |.desiredResourceState.CloudProviderConfig.Aws.RoleId?|=$role |.desiredResourceState.CloudProviderConfig.Aws.IamUserARN?|=$atlasAWSAccountArn |.desiredResourceState.CloudProviderConfig.Aws.ExternalId?|=$atlasAssumedRoleExternalId | .desiredResourceState.CloudProviderConfig.Aws.IamAssumedRoleARN?|=$AWSAssumedArn | .desiredResourceState.ProjectId?|=$projectId' \ - "$(dirname "$0")/datalakes.sample-cfn-request.json" diff --git a/cfn-resources/datalakes/test/datalakes.sample-cfn-request.json b/cfn-resources/datalakes/test/datalakes.sample-cfn-request.json deleted file mode 100644 index deacd6cf4..000000000 --- a/cfn-resources/datalakes/test/datalakes.sample-cfn-request.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "desiredResourceState": { - "Profile": "default", - "CloudProviderConfig": { - "Aws": { - "ExternalId": "", - "IamAssumedRoleARN": "", - "IamUserARN": "", - "RoleId": "", - "TestS3Bucket": "" - } - }, - "ProjectId": "", - "Name": "", - "TenantName": "cfntest", - "DataProcessRegion": { - "CloudProvider": "AWS", - "Region": "" - } - }, - "previousResourceState": {} -} diff --git a/cfn-resources/datalakes/test/datalakes.sample-template.yaml b/cfn-resources/datalakes/test/datalakes.sample-template.yaml deleted file mode 100644 index 50ca71165..000000000 --- a/cfn-resources/datalakes/test/datalakes.sample-template.yaml +++ /dev/null @@ -1,64 +0,0 @@ ---- -AWSTemplateFormatVersion: '2010-09-09' -Description: MongoDB Atlas AWS CloudFormation Quickstart for MongoDB::Atlas::CloudBackUpRestoreJobs -Parameters: - Profile: - Description: "Your MongoDB Cloud Public API Key" - Type: String - Default: "default" - ProjectId: - Description: "Your Project Id" - Type: String - Default: "" - RoleID: - Description: "The name of the Atlas cluster that contains the snapshots you want to retrieve." - Type: String - Default: "" - IamAssumedRoleARN: - Description: "IamAssumedRoleARN." - Type: String - Default: "" - IamUserARN: - Description: "IamUserARN" - Type: String - Default: "" - ExternalId: - Description: "ExternalId" - Type: String - Default: "" - TestS3Bucket: - Description: "TestS3Bucket" - Type: String - Default: "" - TenantName: - Description: "TenantName" - Type: String - Default: "" - Region: - Description: "The name of the Atlas cluster that contains the snapshots you want to retrieve." - Type: String - Default: "MUMBAI_IND" - -Resources: - DataLakes: - Type: MongoDB::Atlas::DataLakes - Properties: - Profile: !Ref Profile - GroupId: !Ref "ProjectId" - TenantName: !Ref "TenantName" - CloudProviderConfig: - Aws: - ExternalId: !Ref "ExternalId" - IamAssumedRoleARN: !Ref "IamAssumedRoleARN" - IamUserARN: !Ref "IamUserARN" - RoleId: !Ref "RoleID" - TestS3Bucket: !Ref "TestS3Bucket" - DataProcessRegion: - CloudProvider: "AWS" - Region: !Ref "Region" -Outputs: - DataLakes: - Description: "Info on DataLakes" - Value: !Ref DataLakes - Export: - Name: !Join [ "-", [ !Ref "AWS::StackName","DataLakes" ] ] \ No newline at end of file diff --git a/cfn-resources/datalakes/test/inputs_1_create.template.json b/cfn-resources/datalakes/test/inputs_1_create.template.json deleted file mode 100644 index 295beec83..000000000 --- a/cfn-resources/datalakes/test/inputs_1_create.template.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "CloudProviderConfig": { - "Aws": { - "ExternalId": "", - "IamAssumedRoleARN": "", - "IamUserARN": "", - "RoleId": "", - "TestS3Bucket": "" - } - }, - "ProjectId": "", - "TenantName": "cfndemo", - "Profile": "default" -} diff --git a/cfn-resources/datalakes/test/inputs_1_update.template.json b/cfn-resources/datalakes/test/inputs_1_update.template.json deleted file mode 100644 index b2b75ec10..000000000 --- a/cfn-resources/datalakes/test/inputs_1_update.template.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "CloudProviderConfig": { - "Aws": { - "ExternalId": "", - "IamAssumedRoleARN": "", - "IamUserARN": "", - "RoleId": "", - "TestS3Bucket": "" - } - }, - "DataProcessRegion": { - "CloudProvider": "AWS", - "Region": "MUMBAI_IND" - }, - "ProjectId": "", - "Profile": "default" -} diff --git a/cfn-resources/datalakes/test/key-policy-template.json b/cfn-resources/datalakes/test/key-policy-template.json deleted file mode 100644 index d1545fee8..000000000 --- a/cfn-resources/datalakes/test/key-policy-template.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:ListBucket", - "s3:GetObject", - "s3:GetObjectVersion", - "s3:GetBucketLocation" - ], - "Resource": [ - "arn:aws:s3:::cfndemo", - "arn:aws:s3:::cfndemo/*" - ] - } - ] -} \ No newline at end of file diff --git a/cfn-resources/datalakes/test/policy.json b/cfn-resources/datalakes/test/policy.json deleted file mode 100644 index 3dc6c8069..000000000 --- a/cfn-resources/datalakes/test/policy.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "s3:ListAccessPointsForObjectLambda", - "ec2:AuthorizeSecurityGroupIngress", - "s3:PutStorageLensConfiguration", - "iam:CreateRole", - "ec2:AttachInternetGateway", - "iam:PutRolePolicy", - "ec2:ReplaceRoute", - "ec2:DescribeTrafficMirrorSessions", - "ec2:UpdateSecurityGroupRuleDescriptionsIngress", - "ec2:AssociateRouteTable", - "ec2:DeleteRouteTable", - "ec2:DescribeInternetGateways", - "ec2:DescribeTrafficMirrorFilters", - "ec2:RevokeSecurityGroupEgress", - "ec2:CreateRoute", - "ec2:CreateInternetGateway", - "ec2:DescribeAccountAttributes", - "ec2:DescribeManagedPrefixLists", - "ec2:DeleteInternetGateway", - "ec2:DescribeNetworkInterfacePermissions", - "ec2:CreateVpcPeeringConnection", - "ec2:DescribeNetworkAcls", - "ec2:DescribeRouteTables", - "ec2:DescribeClientVpnEndpoints", - "ec2:DescribeEgressOnlyInternetGateways", - "iam:GetRole", - "ec2:UpdateSecurityGroupRuleDescriptionsEgress", - "ec2:GetManagedPrefixListEntries", - "s3:PutAccountPublicAccessBlock", - "ec2:DescribeVpcClassicLinkDnsSupport", - "ec2:CreateTags", - "ec2:DescribeVpnConnections", - "ec2:DescribeVpcPeeringConnections", - "s3:ListJobs", - "ec2:CreateRouteTable", - "cloudformation:*", - "ec2:ModifySecurityGroupRules", - "ec2:DescribeVpcEndpointServiceConfigurations", - "ec2:DescribePrefixLists", - "ec2:DescribeVpcClassicLink", - "ec2:RevokeSecurityGroupIngress", - "ec2:DescribeVpcEndpointServicePermissions", - "ec2:DescribeSecurityGroupRules", - "s3:PutAccessPointPublicAccessBlock", - "ec2:DescribeVpcEndpoints", - "s3:CreateJob", - "ec2:CreateSubnet", - "ec2:DescribeSubnets", - "iam:GetRolePolicy", - "ec2:DescribeVpnGateways", - "ec2:DescribeMovingAddresses", - "s3:GetAccessPoint", - "ec2:DescribeAddresses", - "ec2:DescribeFlowLogs", - "ec2:*VpcEndpoint*", - "ec2:CreateVpc", - "ec2:DescribeTransitGateways", - "ec2:DescribeDhcpOptions", - "ec2:*VpcPeeringConnection", - "ec2:DescribeVpcEndpointServices", - "ec2:DescribeVpcAttribute", - "iam:PassRole", - "ec2:DescribeNetworkInterfaces", - "ec2:DescribeTransitGatewayRouteTables", - "ec2:DescribeAvailabilityZones", - "ec2:DescribeTrafficMirrorTargets", - "iam:DeleteRolePolicy", - "kms:*", - "ec2:DescribeNetworkInterfaceAttribute", - "ec2:CreateSecurityGroup", - "ec2:ModifyVpcAttribute", - "ec2:DescribeVpcEndpointConnections", - "ec2:AuthorizeSecurityGroupEgress", - "s3:ListAccessPoints", - "s3:*", - "ec2:DescribeTags", - "ec2:DeleteRoute", - "ec2:DescribeClassicLinkInstances", - "ec2:DescribeNatGateways", - "s3:ListMultiRegionAccessPoints", - "ec2:DescribeCustomerGateways", - "ec2:DescribeVpcEndpointConnectionNotifications", - "ec2:DescribeSecurityGroups", - "s3:ListStorageLensConfigurations", - "s3:GetAccountPublicAccessBlock", - "s3:ListAllMyBuckets", - "ec2:DescribeSecurityGroupReferences", - "ec2:DescribeVpcs", - "ec2:DeleteSecurityGroup", - "ec2:GetManagedPrefixListAssociations", - "ec2:DescribeTransitGatewayVpcAttachments", - "ec2:DescribeStaleSecurityGroups" - ], - "Resource": "*" - } - ] -} \ No newline at end of file diff --git a/cfn-resources/datalakes/test/role-policy-template.json b/cfn-resources/datalakes/test/role-policy-template.json deleted file mode 100644 index 13da4efb7..000000000 --- a/cfn-resources/datalakes/test/role-policy-template.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "AWS": "" - }, - "Action": "sts:AssumeRole", - "Condition": { - "StringEquals": { - "sts:ExternalId": "" - } - } - } - ] -} \ No newline at end of file diff --git a/examples/data-lake-pipeline/datalakepipeline.json b/examples/data-lake-pipeline/datalakepipeline.json deleted file mode 100644 index 4c8fb3745..000000000 --- a/examples/data-lake-pipeline/datalakepipeline.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "AWSTemplateFormatVersion": "2010-09-09", - "Description": "This template helps in creating the Atlas data lake pipeline in the specified project.", - "Parameters": { - "Profile": { - "Type": "String", - "Default": "default" - }, - "ProjectId": { - "Type": "String", - "Description": "The unique ID for the project to create a data lake pipeline.", - "Default": "64cd58d2ee26f172e0a3d648" - }, - "Name": { - "Type": "String", - "Description": "Name of the Atlas Data Lake Pipeline.", - "Default": "datalakepipeline_1" - }, - "FieldName": { - "Type": "String", - "Description": "Identifies the field name used to partition data.", - "Default": "TestpartitionFeildName-1" - }, - "Order": { - "Type": "String", - "Description": "Sequence in which MongoDB Atlas slices the collection data to create partitions. The resource expresses this sequence starting with zero.", - "Default": "0" - }, - "ClusterName": { - "Type": "String", - "Description": "Human-readable name that identifies the cluster.", - "Default": "testdatalakepipeline1" - }, - "CollectionName": { - "Type": "String", - "Description": "Human-readable name that identifies the collection.", - "Default": "test_collection" - }, - "DatabaseName": { - "Type": "String", - "Description": "Human-readable name that identifies the database.", - "Default": "Test-DatabaseName" - }, - "TransformationField": { - "Type": "String", - "Description": "Key in the document.", - "Default": "testtransformFeild" - }, - "Type": { - "Type": "String", - "Description": "Type of transformation applied during the export of the namespace in a Data Lake Pipeline.", - "Default": "EXCLUDE" - } - }, - "Mappings": {}, - "Resources": { - "DataLakePipeline": { - "Type": "MongoDB::Atlas::DataLakePipeline", - "Properties": { - "ProjectId": {"Ref" : "ProjectId"}, - "Name": {"Ref" : "Name"}, - "Profile": {"Ref" : "Profile"}, - "Sink": { - "PartitionFields": [ - { - "FieldName": {"Ref" : "FieldName"}, - "Order": "0" - } - ] - }, - "Source": { - "ClusterName": {"Ref" : "ClusterName"}, - "CollectionName": {"Ref" : "CollectionName"}, - "DatabaseName": {"Ref" : "DatabaseName"} - }, - "Transformations": [ - { - "Field" :{"Ref" : "TransformationField"}, - "Type": "EXCLUDE" - } - ] - } - } - }, - "Outputs": { - "Id": { - "Description": "Id of the project", - "Value": { - "Fn::GetAtt": [ - "DataLakePipeline", - "Id" - ] - } - } - } - } diff --git a/examples/data-lakes/datalake.json b/examples/data-lakes/datalake.json deleted file mode 100644 index 29492e7cd..000000000 --- a/examples/data-lakes/datalake.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "AWSTemplateFormatVersion": "2010-09-09", - "Description": "This template creates an Atlas Data Lake in the specified project", - "Parameters": { - "Profile": { - "Type": "String", - "Default": "default", - "Description": "Secret Manager Profile that contains the Atlas Programmatic keys" - }, - "ProjectId": { - "Type": "String", - "Description": "Atlas Project Id" - }, - "RoleID": { - "Type": "String", - "Description": "Unique identifier of the role that the federated database instance can use to access S3 Buckets. If necessary, use the Atlas API to retrieve the role ID. You must also specify the testS3Bucket", - "Default": "", - "ConstraintDescription": "" - }, - "IamAssumedRoleARN": { - "Type": "String", - "Description": "Amazon Resource Name (ARN) of the IAM Role that the federated database instance assumes when accessing the S3 Buckets", - "Default": "", - "ConstraintDescription": "" - }, - "IamUserARN": { - "Type": "String", - "Description": "Amazon Resource Name (ARN) of the user that the federated database instance uses when accessing the S3 Buckets", - "Default": "", - "ConstraintDescription": "" - }, - "ExternalId": { - "Type": "String", - "Description": "Unique identifier associated with the IAM Role that the federated database instance assumes when accessing the AWS S3 buckets", - "Default": "", - "ConstraintDescription": "" - }, - "TestS3Bucket": { - "Type": "String", - "Description": "", - "Default": "", - "ConstraintDescription": "Name of the S3 data bucket that the provided role ID is authorized to access. You must also specify the roleId" - }, - "TenantName": { - "Type": "String", - "Description": "", - "Default": "", - "ConstraintDescription": "" - }, - "Region": { - "Type": "String", - "Description": "Name of the region to which Data Federation routes client connections for data processing", - "Default": "MUMBAI_IND", - "ConstraintDescription": "" - } - }, - "Mappings": {}, - "Resources": { - "DataLake": { - "Type": "MongoDB::Atlas::DataLakes", - "Properties": { - "ProjectId": { - "Ref": "ProjectId" - }, - "TenantName": { - "Ref": "TenantName" - }, - "CloudProviderConfig": { - "Aws": { - "ExternalId": { - "Ref": "ExternalId" - }, - "IamAssumedRoleARN": { - "Ref": "IamAssumedRoleARN" - }, - "IamUserARN": { - "Ref": "IamUserARN" - }, - "RoleId": { - "Ref": "RoleID" - }, - "TestS3Bucket": { - "Ref": "TestS3Bucket" - } - } - }, - "DataProcessRegion": { - "CloudProvider": "AWS", - "Region": { - "Ref": "Region" - } - }, - "Profile": { - "Ref": "Profile" - } - } - } - }, - "Outputs": { - "DataLake": { - "Description": "Datalake object", - "Value": { - "Fn::Select": [ - 0, - { - "Fn::GetAtt": [ - "DataLake", - "Hostnames" - ] - } - ] - } - } - } -}