Add new hackathon: Agentic Data Cloud for DisneyLand

README.md

@@ -38,6 +38,8 @@ Teams are not left on their own to solve the challenges. Coaches work with each
   > We will be going through replicating and processing operational data from an Oracle database into Google Cloud in real time. You'll also figure out how to forecast future demand, and how to visualize this forecast data as it arrives.
 - [Hack to the Future: Data Track](./hacks/httf-data/README.md)
   > In this hack we'll help the fictitious Cymbal Shops e-commerce platform to modernize their tech stack, primarily focusing on the migration of the legacy application MySQL database to Cloud Spanner. From that point we'll create BigQuery analytics datasets to handle both historical and live data using federation, use advanced generative AI to enhance the existing dataset with descriptions and images, and finally implement semantic search on this dataset.
+- [Disneyland Agentic Data Cloud](./hacks/disneyland-on-gcp/README.md)
+  > Build an end-to-end intelligent guest assistance system. You will ingest operational data into AlloyDB and generate vector embeddings natively using AlloyDB AI, configure QueryData for predictable SQL generation, and set up real-time replication to BigQuery using Datastream. In BigQuery, you will train forecasting models using BigQuery ML, build a multimodal RAG pipeline on images and PDF brochures, and map visitor movement patterns using BigQuery's native SQL Graph (GQL) capabilities. After cataloging your assets in Knowledge Catalog, you will use Conversational Analytics to query insights, sync these insights back to AlloyDB via BigQuery FDW, expose operational and analytical tools using the MCP Toolbox, and build and deploy a conversational assistant web app powered by the Agent Development Kit (ADK).
 
 ### Infrastructure
 

hacks/disneyland-on-gcp/QL_OWNER

@@ -0,0 +1,3 @@
+meken@google.com
+# Collaborators
+ginof@google.com

hacks/disneyland-on-gcp/artifacts/Makefile

@@ -0,0 +1,21 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+TARGET=ghacks-setup.zip
+OBJS=*.tf runtime.yaml
+
+$(TARGET): $(OBJS)
+	zip -r $(TARGET) $(OBJS)
+
+clean:
+	rm -f $(TARGET)

hacks/disneyland-on-gcp/artifacts/main.tf

@@ -0,0 +1,261 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource "google_project_service" "default" {
+  project = var.gcp_project_id
+  for_each = toset([
+    "cloudresourcemanager.googleapis.com",
+    "serviceusage.googleapis.com",
+    "iam.googleapis.com",
+    "storage.googleapis.com",
+    "compute.googleapis.com",
+    "alloydb.googleapis.com",
+    "bigquery.googleapis.com",
+    "bigqueryconnection.googleapis.com",
+    "datastream.googleapis.com",
+    "aiplatform.googleapis.com",
+    "dataplex.googleapis.com",
+    "artifactregistry.googleapis.com",
+    "run.googleapis.com",
+    "servicenetworking.googleapis.com",
+    "cloudaicompanion.googleapis.com",
+    "geminicloudassist.googleapis.com",
+    "geminidataanalytics.googleapis.com"
+  ])
+  service = each.key
+
+  disable_on_destroy = false
+}
+
+# In case a default network is not present in the project, this variable needs to be set
+resource "google_compute_network" "default_network_created" {
+  name                    = "default"
+  auto_create_subnetworks = true
+  count                   = var.create_default_network ? 1 : 0
+  depends_on = [
+    google_project_service.default
+  ]
+}
+
+# Enabling comms between VMs for auto mode subnets (needed by Dataproc/Dataflow workers)
+resource "google_compute_firewall" "fwr_allow_custom" {
+  name          = "fwr-ingress-allow-custom"
+  network       = google_compute_network.default_network_created[0].self_link
+  count         = var.create_default_network ? 1 : 0
+  source_ranges = ["10.128.0.0/9"]
+  allow {
+    protocol = "all"
+  }
+}
+
+# Enabling Identity-Aware-Proxy for TCP forwarding (for SSH access from Console)
+resource "google_compute_firewall" "fwr_allow_iap" {
+  name          = "fwr-ingress-allow-iap"
+  network       = google_compute_network.default_network_created[0].self_link
+  count         = var.create_default_network ? 1 : 0
+  source_ranges = ["35.235.240.0/20"]
+  allow {
+    protocol = "tcp"
+    ports    = ["22"]
+  }
+}
+
+# This piece of code makes it possible to deal with the default network the same way, 
+# regardless of how it has been created. Make sure to refer to the default network through
+# this resource when needed.
+data "google_compute_network" "default_network" {
+  name = "default"
+  depends_on = [
+    google_project_service.default,
+    google_compute_network.default_network_created
+  ]
+}
+
+# Reserve a range for Private Service Access
+resource "google_compute_global_address" "private_ip_alloc" {
+  name          = "alloydb-private-ip-alloc"
+  purpose       = "VPC_PEERING"
+  address_type  = "INTERNAL"
+  prefix_length = 16
+  network       = data.google_compute_network.default_network.id
+}
+
+# Create the VPC Peering connection
+resource "google_service_networking_connection" "vpc_connection" {
+  network                 = data.google_compute_network.default_network.id
+  service                 = "servicenetworking.googleapis.com"
+  reserved_peering_ranges = [google_compute_global_address.private_ip_alloc.name]
+  depends_on              = [google_project_service.default]
+}
+
+# Random password for the AlloyDB database user
+resource "random_string" "password" {
+  length  = 12
+  special = false
+}
+
+# AlloyDB Cluster
+resource "google_alloydb_cluster" "default" {
+  cluster_id = "disney-cluster"
+  location   = var.gcp_region
+
+  network_config {
+    network = data.google_compute_network.default_network.id
+  }
+
+  initial_user {
+    user     = "postgres"
+    password = random_string.password.result
+  }
+
+  depends_on = [
+    google_project_service.default,
+    google_service_networking_connection.vpc_connection
+  ]
+}
+
+resource "google_alloydb_instance" "default" {
+  cluster       = google_alloydb_cluster.default.name
+  instance_id   = "disney-instance"
+  instance_type = "PRIMARY"
+
+  database_flags = {
+    "alloydb.logical_decoding" = "on" # needed for replication
+    "bigquery_fdw.enabled"     = "on" # needed for connecting to bq
+    "alloydb.iam_authentication" = "on"
+  }
+
+  machine_config {
+    cpu_count = 2
+  }
+
+  network_config {
+    enable_public_ip = true
+    authorized_external_networks {
+      cidr_range = "0.0.0.0/0"
+    }
+  }
+
+  depends_on = [google_alloydb_cluster.default]
+}
+
+# Storage Bucket
+resource "google_storage_bucket" "disney" {
+  name                        = var.gcp_project_id
+  location                    = var.gcp_region
+  uniform_bucket_level_access = true
+}
+
+# BigQuery Dataset
+resource "google_bigquery_dataset" "disney" {
+  dataset_id                 = "disney"
+  friendly_name              = "Disneyland Data"
+  description                = "Dataset for Disneyland Data Cloud"
+  location                   = var.gcp_region
+  delete_contents_on_destroy = true
+
+  depends_on = [google_project_service.default]
+}
+
+resource "google_bigquery_reservation" "bq_edition" {
+  name          = "disneyland-analytics"
+  location      = var.gcp_region
+  edition       = "ENTERPRISE"
+  slot_capacity = 0
+
+  autoscale {
+    max_slots = 100
+  }
+}
+
+resource "google_bigquery_reservation_assignment" "default" {
+  assignee    = "projects/${var.gcp_project_id}"
+  reservation = google_bigquery_reservation.bq_edition.id
+  job_type    = "QUERY"
+}
+
+# BigQuery Cloud Resource Connection
+resource "google_bigquery_connection" "agent_platform" {
+  connection_id = "conn"
+  location      = var.gcp_region
+  friendly_name = "Connection to Agent Platform"
+  cloud_resource {}
+
+  depends_on = [google_project_service.default]
+}
+
+# Grant Agent Platform User and Storage Viewer to BigQuery Connection Service Account
+resource "google_project_iam_member" "bq_connection_roles" {
+  for_each = toset([
+    "roles/aiplatform.user",
+    "roles/storage.objectViewer"
+  ])
+  project = var.gcp_project_id
+  role    = each.key
+  member  = "serviceAccount:${google_bigquery_connection.agent_platform.cloud_resource[0].service_account_id}"
+}
+
+# AlloyDB Service Agent
+resource "google_project_service_identity" "alloydb_sa" {
+  provider = google-beta
+  project  = var.gcp_project_id
+  service  = "alloydb.googleapis.com"
+}
+
+# Grant Agent Platform User, BQ Viewer and BQ Read Session User to AlloyDB Service Agent
+resource "google_project_iam_member" "alloydb_sa_roles" {
+  for_each = toset([
+    "roles/aiplatform.user",
+    # "roles/bigquery.dataViewer",  # these need to be granted to another service account
+    # "roles/bigquery.readSessionUser"
+  ])
+  project = var.gcp_project_id
+  role    = each.key
+  member  = "serviceAccount:${google_project_service_identity.alloydb_sa.email}"
+}
+
+# =========================================================================
+# Dedicated SA for the agent & QueryData
+# =========================================================================
+
+# 1. Create the Service Account
+resource "google_service_account" "agent_sa" {
+  account_id   = "disney-agent-sa"
+  display_name = "Disneyland Guest Assistant Agent Service Account"
+}
+
+# 2. Grant required roles to the Service Account
+resource "google_project_iam_member" "agent_sa_roles" {
+  for_each = toset([
+    "roles/geminidataanalytics.queryDataUser",
+    "roles/alloydb.databaseUser",
+    "roles/serviceusage.serviceUsageConsumer",
+    "roles/aiplatform.user"
+  ])
+  project = var.gcp_project_id
+  role    = each.key
+  member  = "serviceAccount:${google_service_account.agent_sa.email}"
+}
+
+# 3. Allow all Project Editors and Owners to impersonate this Service Account
+resource "google_service_account_iam_binding" "sa_impersonation" {
+  service_account_id = google_service_account.agent_sa.name
+  role               = "roles/iam.serviceAccountTokenCreator"
+
+  members = [
+    "projectEditor:${var.gcp_project_id}",
+    "projectOwner:${var.gcp_project_id}"
+  ]
+}
+

hacks/disneyland-on-gcp/artifacts/outputs.tf

@@ -0,0 +1,30 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+output "project_id" {
+  value = var.gcp_project_id
+}
+
+output "alloydb_proxy_ip" {
+  value = google_compute_instance.gce_tcp_proxy.network_interface[0].access_config[0].nat_ip
+}
+
+output "alloydb_usr" {
+  value = google_alloydb_cluster.default.initial_user[0].user
+}
+
+output "alloydb_pwd" {
+  value     = google_alloydb_cluster.default.initial_user[0].password
+  sensitive = true
+}

hacks/disneyland-on-gcp/artifacts/providers.tf

@@ -0,0 +1,37 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 7.24.0"
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = "~> 7.24.0"
+    }
+  }
+}
+
+provider "google" {
+  project = var.gcp_project_id
+  region  = var.gcp_region
+  zone    = var.gcp_zone
+}
+
+provider "google-beta" {
+  project = var.gcp_project_id
+  region  = var.gcp_region
+  zone    = var.gcp_zone
+}

hacks/disneyland-on-gcp/artifacts/runtime.yaml

@@ -0,0 +1,15 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+runtime: terraform
+version: 1.4.6