diff --git a/.gitignore b/.gitignore index 1a3e321..a6a0ea5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ -token.json \ No newline at end of file +token.json +__pycache__/ +*.pyc +*.egg-info/ diff --git a/.vscode/launch.json b/.vscode/launch.json index 6b76b4f..fc299dd 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,6 +10,15 @@ "request": "launch", "program": "${file}", "console": "integratedTerminal" + }, + { + "name": "Pytest: Current File", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": ["${file}", "-v", "-s"], + "console": "integratedTerminal", + "cwd": "${fileDirname}" } ] } \ No newline at end of file diff --git a/repoScaffold/examples/ai4us2.yaml b/repoScaffold/examples/ai4us2.yaml new file mode 100644 index 0000000..c4b1a5c --- /dev/null +++ b/repoScaffold/examples/ai4us2.yaml @@ -0,0 +1,54 @@ +project: + name: ai4us2 + env: dev + cloud: + provider: gcp + region: us-central1 + project_id: ai4us2 + +database: + type: mongodb + atlas_cluster: Cluster0 + db_name: ai4us2 + +services: + - type: api + enabled: true + name: api + subdomain: api + stack: express + port: 3006 + + - type: webapp + enabled: true + name: app + subdomain: app + stack: react + port: 3005 + + - type: worker + enabled: true + name: worker + stack: python + port: 8080 + gpu: required + +agents: + service_accounts: + - 464961297779-compute@developer.gserviceaccount.com + roles: + - name: developer + sleep_seconds: 600 + max_turns: 1000 + - name: reviewer + sleep_seconds: 1800 + max_turns: 200 + - name: ops + sleep_seconds: 1800 + max_turns: 100 + +linear: + enabled: true + workspace: ai4us + team_key: AI4 + team_name: AI4US diff --git a/repoScaffold/examples/carouselr.yaml b/repoScaffold/examples/carouselr.yaml new file mode 100644 index 0000000..0b23fb6 --- /dev/null +++ b/repoScaffold/examples/carouselr.yaml @@ -0,0 +1,36 @@ +project: + name: carouselr + env: dev + cloud: + provider: gcp + region: us-central1 + project_id: carouselr-app + +# Browser-only app — disable everything except the webapp. +# Keep DB block but disable Phase C; no Mongo connection needed. +database: + type: mongodb + atlas_cluster: Cluster0 + db_name: carouselr + +services: + - type: api + enabled: false + name: api + stack: express + port: 3006 + + - type: webapp + enabled: true + name: app + stack: react + port: 3005 + + - type: worker + enabled: false + name: worker + stack: python + port: 8080 + +linear: + enabled: false diff --git a/repoScaffold/examples/crud_app.yaml b/repoScaffold/examples/crud_app.yaml new file mode 100644 index 0000000..d8f9f2b --- /dev/null +++ b/repoScaffold/examples/crud_app.yaml @@ -0,0 +1,31 @@ +project: + name: MyApp + env: dev + cloud: + provider: gcp + region: us-central1 + project_id: MyApp # Should default to project name + +database: + type: mongodb + atlas_cluster: Cluster0 + db_name: my_app + +services: + - type: api + enabled: true + name: api + subdomain: api # should be automatic based + stack: express + port: 3006 # API should default to 3006 + + - type: webapp + enabled: true + name: app + subdomain: app # default to name + stack: react + port: 3005 # Web should default to 3005 + + - type: worker # Should use Cloud run function by default + enabled: false + gpu: optional \ No newline at end of file diff --git a/repoScaffold/examples/fresler-table.yaml b/repoScaffold/examples/fresler-table.yaml new file mode 100644 index 0000000..ede1949 --- /dev/null +++ b/repoScaffold/examples/fresler-table.yaml @@ -0,0 +1,34 @@ +project: + name: fresler-table + kind: library-node # publishable npm package — no services/infra/deploy + env: dev + repo: gas6262/fresler-table # existing GitHub repo (for the Cloud Build trigger) + cloud: + provider: gcp + # Dedicated GCP project: single home for this library's pipeline + agent. + project_id: fresler-table + +# The publishable package lives in src/ (not the repo root) and builds with tsup. +library: + package_dir: src + build_cmd: npm run build + test_cmd: npm test + publish_cmd: npm publish + registry_url: https://www.npmjs.com/package/@fresler/fresler-table + +# No services, database, or infra — this is a library. +services: [] + +agents: + service_accounts: + - 464961297779-compute@developer.gserviceaccount.com + roles: + - name: developer + sleep_seconds: 900 + max_turns: 600 + +linear: + enabled: true + workspace: fresler # shared workspace (rename to a common slug later) + team_key: FT + team_name: Fresler Table diff --git a/repoScaffold/prompt.md b/repoScaffold/prompt.md new file mode 100644 index 0000000..0f041b9 --- /dev/null +++ b/repoScaffold/prompt.md @@ -0,0 +1,377 @@ +Create an opinionated Python-based CLI that scaffolds, provisions, tests, and deploys a full-stack application platform end-to-end on GCP for a development environment. + +## Goal + +Build a highly modular, readable, agent-friendly CLI that can create a new project from scratch and get it working end-to-end with minimal manual steps. The CLI must be designed so that either a human or coding agent can safely inspect, extend, and iterate on it. + +The CLI should favor reusing existing mature CLIs and tools whenever possible rather than reimplementing behavior from scratch. + +The primary command should be: + +```bash +platform-cli scaffold [options] +``` + +The CLI should also support: + +```bash +platform-cli install # Install all dependencies and CLIs needed +platform-cli test # Test specific service +``` + +## Core Requirements + +### 1. Language / Architecture + +* CLI must be Python-based. +* Use a modular architecture with clear package boundaries. +* Code must be highly readable and easy for agents to modify. +* Every major action must be implemented as an independent step/task. +* Steps must be resumable, skippable, and idempotent. +* Use a DAG / graph execution model for scaffold steps. +* Every step should define: + + * inputs + * outputs + * dependencies + * retry behavior + +### 2. Development Environment (v1 scope) + +* Only support dev environment initially. +* Everything must work locally first. +* Generated local setup must work with developer credentials. +* Local apps can run outside Docker for fast iteration. + +### 3. Opinionated Defaults + +#### Local Ports + +* Frontend always runs on localhost:3005 +* API always runs on localhost:3006 +* Docker containers should expose port 80 externally wherever practical. + +#### Cloud + +* GCP is required. However, it should still use Terraform for config-driven infrastructure +* All deployable services should run on Cloud Run or Cloud Run Jobs / Functions where appropriate. +* Strong preference for Docker containerization everywhere. + +### 4. Project Manifest (Source of Truth) + +Generate a manifest file that defines the full project: + +Example: + +```yaml +project: + name: MyApp + env: dev + cloud: + provider: gcp + region: us-central1 + project_id: MyApp # Should default to project name + +database: + type: mongodb + atlas_cluster: Cluster0 + db_name: my_app + +services: + - type: api + enabled: true + name: api + subdomain: api # should be automatic based + stack: express + port: 3006 # API should default to 3006 + + - type: webapp + enabled: true + name: app + subdomain: app # default to name + stack: react + port: 3005 # Web should default to 3005 + + - type: worker # Should use Cloud run function by default + enabled: false + gpu: optional +``` + +This manifest must drive all generated artifacts. + +## Required Scaffold Scope + +## Phase A: Bootstrap / Tooling + +### Tool Detection / Installation + +CLI should detect / install: + +* gcloud CLI +* terraform +* docker +* mongodb atlas CLI (if MongoDB selected) +* node / npm if needed + +This is a separate install command. User should download ALL tooling as a prerequisite rather than autodetection. Must provide clear instructions if installation requires user approval. + +## Phase B: Repo / Folder Scaffold + +Generate: + +```text +project/ + cli/ + services/ + api/ + src/ + DockerFile + .env + ... + web/ + worker/ + infra/ + terraform/ + modules/ + envs/dev/ + cloudbuild/ + terraform.yaml # If necessary + api.yaml + web.yaml + cloudrun/ + api.yaml + web.yaml + .vscode/ + .claude/ + AGENTS.md + .env.example + .gitignore + docker-compose.yml + project.manifest.yaml +``` + +### Additional Files + +Must generate: + +* AGENTS.md with architecture + conventions +* .claude config / guidance +* .vscode launch/tasks for local run/test +* README with setup steps + +## Phase C: Database Setup + +### MongoDB Atlas Support (initial DB support) + +If DB type = mongodb: + +CLI should: + +* authenticate with Atlas CLI +* select org / project +* inspect Cluster0 +* create DB if missing +* skip if exists +* create dummy collection: items +* insert seed rows / docs + +Example seed: + +```json +[{"name": "example item"}] +``` + +### DB Credentials + +After DB setup: + +* generate connection string +* store locally in a local secrets manifest such as .env +* then store in GCP Secret Manager for the project +* maintain secrets manifest + +Secret sync should be idempotent. + +## Phase D: API Scaffold + +### Initial API Support + +Support only: + +* Node.js Express API + +If DB = MongoDB: + +* auto-add Mongoose +* create DB connection layer +* create model for items + +Generated API must include: + +* GET /items + +Requirements: + +* API works immediately locally. +* API reads credentials from env. +* proper folder structure. +* lint config. +* Dockerfile. + +### Docker Requirements + +* every service dockerized. +* API Dockerfile maps to port 80. +* Dockerfiles should follow best practices. + +## Phase E: Frontend Scaffold + +Initial support: + +* React + +Requirements: + +* reuse standard scaffold CLI if possible. +* homepage loads. +* can fetch /items. +* local runs on 3005. +* Dockerfile. + +Frontend should not need knowledge of API implementation details. +Only use configured endpoint contracts (initially /items). + +## Phase F: Worker Scaffold + +Requirements: + +* Cloud Run compatible. +* optional GPU support config. +* scaffold stub worker. +* dockerized. + +## Phase G: GCP Setup + +CLI should support: + +* create or connect GCP project +* enable required APIs: + * Cloud Run + * Cloud Build + * Secret Manager + * Artifact Registry +* create Artifact Registry repo +* create service accounts +* configure IAM minimally + +Must clearly separate: + +* build service account +* runtime service account + +## Phase H: Secret Manager + +Create a secret registry system. + +Requirements: + +* secret manifest file +* sync local env -> GCP Secret Manager +* generate .env.example comments for secret-backed vars +* never commit local secrets + +## Phase I: Cloud Build + +Generate Cloud Build configs for: + +* api build / deploy +* web build / deploy +* terraform rollout + +Requirements: + +* Cloud Build injects secrets from Secret Manager +* service builds are modular +* Terraform has ONE dedicated pipeline + +## Phase J: Terraform + +Generate Terraform for: + +* Cloud Run services +* IAM +* secret access +* networking basics if needed + +Requirements: + +* modular TF structure +* dev environment only initially +* single Terraform Cloud Build pipeline +* Terraform references generated Cloud Run configs where useful + +## Phase K: Testing + +CLI test command must support: + +### API tests + +* lint +* local start +* health check +* GET /items +* docker build test +* docker run smoke test + +### Frontend tests + +* local start +* page load smoke test + +### Infra tests + +* terraform validate +* terraform plan + +### Overall + +* clear logs +* fail fast +* retry support + +## Quality Constraints + +* prioritize correctness over speed +* highly modular and extensible +* no tightly coupled assumptions +* services communicate via contracts only +* every generated file should be production-sensible even if dev-first +* generated code should be clean and not toy quality + +## Important Build Strategy + +Do NOT try to make everything perfect in one pass. +Instead: + +* scaffold a clean first version +* run local validations automatically +* detect failures +* iteratively fix generated code until: + * local API works + * DB connects + * /items returns data + * docker builds succeed + +The CLI itself should be designed to support this iterative agent workflow. + +## Deliverables + +Build: + +* the Python CLI codebase +* generated project template system +* all config files +* tests +* docs + +Focus on making the CLI itself robust, modular, and easy to evolve. diff --git a/repoScaffold/promptToPrompt.md b/repoScaffold/promptToPrompt.md new file mode 100644 index 0000000..b48db10 --- /dev/null +++ b/repoScaffold/promptToPrompt.md @@ -0,0 +1,43 @@ +This is a great start. Let's refine the prompt a bit more based on these items + +the CLI should be Python based, readable and modular so I or an agent can easily make changes intuitively without breaking other aspects of the app + +It needs to also setup the new project in GCP. All APIs and apps should run in Cloudrun and everything should have a big focus on docker containerization and using the same port number for most HTTP requests to make everything simple. It should be highly opinionated. Everything should be dockerized out of the box. + +It needs to go a bit further than just setting up the repo as well. It needs to set up and all of the new infrastructure end to end. We need to expand a bit on the services schema. When I specify an API, it should be able to specify the stack for the API as well. Same for the worker and front end. + +I should also be able to set up a database (just as you enabled setting up services). If my db type is set to mongodb, it should set up the db inside of cluster 0 in my db using my mongodb Atlas credentials . inside cluster zero in my Atlas configuration. If it doesn't exist, then it should create it. If it does exist, then it should skip. All of the items need to be connected together such that I can do a full end to end hello world at the beginning. It should create a table in the new database called items with a basic schema such as just one column such as or name With just one entry in the table or a small set of entries using my user credentials. + +For now lets only support the dev environment (and running locally using dev credentials) + +Everything needs to work e2e and it should generate in order. When I have a mongodb, it needs to generate the new database using atlas if necessary via CLI. It should download any atlas CLI as necessary to do this. After generating the db (if it doesn't exist), it needs to generate a credentials or connections strings for the db which will be stored 2 places. On the local machine in the env files needed by the API or any downstream dependencies as well as in the GCP secret manager using the gcp CLI or any necessary command line tools The secrets should have the same name and there should be a commented section in the env file for all variables that live in the projects secret store. Then it should create the API using the stack I specify (default express + ORM based on db - mongoose for mongodb). Given the previously obtained credentials, the API should already be a hello world API with a GET /items endpoint that can obtain the items from the database. This should work on first load. + +After creating the API, it should then create the corresponding docker files for the API which should map HTTP to port 80 as should all of my dockerfiles regardless of the original port of the app etc. Dockerfiles should contain the necessary requirements to build and run the app based on the stack (of the API or the app). + +After creating the dockerfiles, it should generate the cloudbuild files for the API (or later the webb app) should be generated. All of the secrets that were needed will have been added to the secret store in GCP for the project and should be referenced in the .env file which should never be pushed to the repo since they will be injected by Cloudbuild as environment variables. + +It then needs to generate terraform files for the necessary infra that can be rolled out. While there may be many terraform files, there should be only one Cloudbuild pipeline for the terraform files. So this should be explicitly listed in your folder breakdown that you had earlier. + +After creating all of those files, it needs to connect everything to GCP. This will work on my local machine with my credentials. For example it needs to create the new Cloudrun instances and point it to the current repo. I've attached an example + +Each service type should be using Cloudrun or Cloudrun functions. There may need to be a separate folder for all the yaml Cloudrun configurations for those services as well. Terraform should be referencing these files when rolling out the infrastructure + +All of the integrations should be as modular as possible. Ideally, the webapp should not need to know the stack of the API. Just that it should exist. When creating the API, there should always be a datbase so when creating the API, we should pass the database credentials as an object prameter and each step should be independent in that way and could be represented as a graph of steps. + +On the local machine, the apps may be run outside of docker for testing and development. webapps should ALWAYS run on localhost:3005 and APIs should always run on localhost:3006. They should not need to know much else about each other other than the endpoints needed (such as /items). + +Workers should be based on Cloudrun. I should be able to specify a gpu type as well even for Cloud run in case the jobs include creating models etc. + +Aside from the folders you listed originally, it also needs to create an agent.md or agents.md file that will provide the agent context on the new scaffold as well as a .claude, and a .vscode folder with all of the startup functions needed to run the projects locally such as running npm start. + +Reuse CLIs as much as ppssible rather than scripting the steps themselves. For example, start with a react init script if one exists. + +There may be many CLIs that are needed to run this. Therefore the cli needs to have + +The CLI needs to be usable and iterateble by an agen such as usage within a skill so build it accordingly + +All necessary credentials needed to make the app work e2e should be stored in GCP secret manager. APIs and UIs should be containerized. When building thei + +When specity an API, i also want to be able to specify the stack for the API and it should generate it consistently. For now let's start with the APIs based on express. If the database is mongodb, it should add a mongoose layer that connects to the + +I think its OK if everything doesn't work the first time. I will work locally with an agent to iterate on the CLI given a few test projects. The CLI needs to have a scaffold command that will do all of this. But I think the CLI also needs to have a test command that will for each service, test every aspect of it. e.g for the API, it should start a process to test the docker build, in parallel start a process to start the API and run the /items or dummy endpoint. It should be able to test the . It should be able to test the webapp very basically - get the webapp home page text, etc. \ No newline at end of file diff --git a/repoScaffold/pyproject.toml b/repoScaffold/pyproject.toml new file mode 100644 index 0000000..7828880 --- /dev/null +++ b/repoScaffold/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "platform-cli" +version = "0.1.0" +description = "CLI to scaffold, provision, test, and deploy full-stack apps on GCP" +requires-python = ">=3.11" +dependencies = [ + "click>=8.1", + "pyyaml>=6.0", + "jinja2>=3.1", + "pydantic>=2.0", + "rich>=13.0", +] + +[project.scripts] +platform-cli = "platform_cli.cli:cli" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +platform_cli = ["../templates/**/*.j2"] diff --git a/repoScaffold/src/platform_cli/__init__.py b/repoScaffold/src/platform_cli/__init__.py new file mode 100644 index 0000000..2d8ef69 --- /dev/null +++ b/repoScaffold/src/platform_cli/__init__.py @@ -0,0 +1 @@ +"""platform-cli: scaffold, provision, test, and deploy full-stack apps on GCP.""" diff --git a/repoScaffold/src/platform_cli/__main__.py b/repoScaffold/src/platform_cli/__main__.py new file mode 100644 index 0000000..ad09b38 --- /dev/null +++ b/repoScaffold/src/platform_cli/__main__.py @@ -0,0 +1,4 @@ +"""Allow running as: python -m platform_cli""" +from platform_cli.cli import cli + +cli() diff --git a/repoScaffold/src/platform_cli/cli.py b/repoScaffold/src/platform_cli/cli.py new file mode 100644 index 0000000..cb5f074 --- /dev/null +++ b/repoScaffold/src/platform_cli/cli.py @@ -0,0 +1,21 @@ +"""Click CLI group: scaffold, install, test.""" +from __future__ import annotations + +import click + +from platform_cli.commands.scaffold import scaffold +from platform_cli.commands.install import install +from platform_cli.commands.test_cmd import test +from platform_cli.commands.create_agent_identity import create_agent_identity + + +@click.group() +@click.version_option(version="0.1.0", prog_name="platform-cli") +def cli() -> None: + """Scaffold, provision, test, and deploy full-stack apps on GCP.""" + + +cli.add_command(scaffold) +cli.add_command(install) +cli.add_command(test) +cli.add_command(create_agent_identity) diff --git a/repoScaffold/src/platform_cli/commands/__init__.py b/repoScaffold/src/platform_cli/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/repoScaffold/src/platform_cli/commands/create_agent_identity.py b/repoScaffold/src/platform_cli/commands/create_agent_identity.py new file mode 100644 index 0000000..151b595 --- /dev/null +++ b/repoScaffold/src/platform_cli/commands/create_agent_identity.py @@ -0,0 +1,176 @@ +"""platform-cli create-agent-identity — give a project's agent its own GitHub App. + +Automates the whole GitHub App manifest flow so each scaffolded project gets its +own scoped bot identity (`[bot]`) instead of borrowing a human's token: + + 1. serve a pre-filled App manifest locally + open the browser + 2. you click "Create GitHub App" (the one consent GitHub requires) + 3. capture the redirect code, exchange it for the App id + private key + 4. store the private key in the project's Secret Manager + 5. open the install page + wait until the App is installed on the repo + 6. resolve the installation id + bot commit identity and print the wiring + +The printed values (app_id, installation_id, bot_name, bot_email) go straight +into the generated agents/loop.sh (see the loop template's App-auth block). +""" +from __future__ import annotations + +import http.server +import json +import socketserver +import subprocess +import tempfile +import threading +import time +import urllib.request +import webbrowser +from pathlib import Path + +import click +from rich.console import Console + +console = Console() +PORT = 8722 +_captured = {"code": None} + + +def _b64url(data: bytes) -> str: + import base64 + return base64.urlsafe_b64encode(data).rstrip(b"=").decode() + + +def _app_jwt(app_id: str, pem: str) -> str: + """Sign an RS256 App JWT using openssl (no extra Python crypto dep).""" + now = int(time.time()) + header = _b64url(json.dumps({"alg": "RS256", "typ": "JWT"}).encode()) + payload = _b64url(json.dumps({"iat": now - 60, "exp": now + 540, "iss": app_id}).encode()) + signing_input = f"{header}.{payload}".encode() + with tempfile.NamedTemporaryFile("w", suffix=".pem", delete=False) as f: + f.write(pem) + key_path = f.name + try: + sig = subprocess.run( + ["openssl", "dgst", "-sha256", "-sign", key_path], + input=signing_input, capture_output=True, check=True, + ).stdout + finally: + Path(key_path).unlink(missing_ok=True) + return f"{header}.{payload}.{_b64url(sig)}" + + +def _gh(url: str, token: str, bearer: bool = False, method: str = "GET", body: dict | None = None) -> dict: + auth = f"Bearer {token}" if bearer else f"token {token}" + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(url, data=data, method=method, headers={ + "Authorization": auth, "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", "User-Agent": "platform-cli", + }) + with urllib.request.urlopen(req) as r: + return json.load(r) + + +def _serve_manifest(manifest: dict) -> threading.Thread: + form = ( + "" + "

Create the agent GitHub App

" + "

Clicking below sends a pre-filled manifest to GitHub " + "(name + Contents/Pull-requests write already set). Review and click " + "Create GitHub App.

" + "
" + f"" + "" + "
" + ) + + class H(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def do_GET(self): + import urllib.parse + parsed = urllib.parse.urlparse(self.path) + if parsed.path == "/created": + code = urllib.parse.parse_qs(parsed.query).get("code", [""])[0] + _captured["code"] = code + msg = b"

App created. Return to your terminal.

" + else: + msg = form.encode() + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.end_headers() + self.wfile.write(msg) + + socketserver.TCPServer.allow_reuse_address = True + httpd = socketserver.TCPServer(("127.0.0.1", PORT), H) + t = threading.Thread(target=httpd.serve_forever, daemon=True) + t.start() + t.httpd = httpd # type: ignore + return t + + +@click.command("create-agent-identity") +@click.option("--project", required=True, help="GCP project holding the agent's secrets.") +@click.option("--repo", required=True, help="GitHub repo as owner/name.") +@click.option("--name", default=None, help="App name (default: -agent).") +@click.option("--secret", default="github-app-private-key", help="Secret Manager name for the App private key.") +def create_agent_identity(project: str, repo: str, name: str | None, secret: str) -> None: + """Create + install a per-project GitHub App bot for the agent.""" + owner, repo_name = repo.split("/", 1) + app_name = name or f"{repo_name}-agent" + manifest = { + "name": app_name, + "url": f"https://github.com/{owner}", + "hook_attributes": {"url": "https://example.com/unused", "active": False}, + "redirect_url": f"http://localhost:{PORT}/created", + "public": False, + "default_permissions": {"contents": "write", "pull_requests": "write", "metadata": "read"}, + "default_events": [], + } + + server = _serve_manifest(manifest) + url = f"http://localhost:{PORT}/" + console.print(f"\n[bold]Opening[/bold] {url} — click [bold]Create GitHub App[/bold] (rename if you like).") + webbrowser.open(url) + + console.print(" [dim]waiting for you to create the App…[/dim]") + while not _captured["code"]: + time.sleep(1) + server.httpd.shutdown() # type: ignore + + # Exchange the manifest code for the App credentials. + conv = _gh(f"https://api.github.com/app-manifests/{_captured['code']}/conversions", + token="", method="POST") + app_id, slug, pem = str(conv["id"]), conv["slug"], conv["pem"] + console.print(f" [green]created[/green] App {slug} (id {app_id})") + + # Store the private key in Secret Manager. + if not subprocess.run(["gcloud", "secrets", "describe", secret, f"--project={project}"], + capture_output=True).returncode == 0: + subprocess.run(["gcloud", "secrets", "create", secret, f"--project={project}", + "--replication-policy=automatic"], check=True, capture_output=True) + subprocess.run(["gcloud", "secrets", "versions", "add", secret, f"--project={project}", + "--data-file=-"], input=pem.encode(), check=True, capture_output=True) + console.print(f" [green]stored[/green] private key in Secret Manager: {secret}") + + # Install + wait for it. + install_url = f"https://github.com/apps/{slug}/installations/new" + console.print(f"\n [bold]Opening[/bold] {install_url} — install on [bold]{repo}[/bold] (Only select repositories).") + webbrowser.open(install_url) + console.print(" [dim]waiting for the install…[/dim]") + install_id = None + while not install_id: + time.sleep(2) + jwt = _app_jwt(app_id, pem) + for inst in _gh("https://api.github.com/app/installations", jwt, bearer=True): + install_id = inst["id"] + break + + bot = _gh(f"https://api.github.com/users/{slug}%5Bbot%5D", _app_jwt(app_id, pem), bearer=True) + bot_name = f"{slug}[bot]" + bot_email = f"{bot['id']}+{slug}[bot]@users.noreply.github.com" + + console.print("\n[bold green]Agent identity ready.[/bold green] Wire these into agents/loop.sh:") + console.print(f" GITHUB_APP_ID=\"{app_id}\"") + console.print(f" GITHUB_APP_INSTALLATION_ID=\"{install_id}\"") + console.print(f" GITHUB_BOT_NAME=\"{bot_name}\"") + console.print(f" GITHUB_BOT_EMAIL=\"{bot_email}\"") diff --git a/repoScaffold/src/platform_cli/commands/install.py b/repoScaffold/src/platform_cli/commands/install.py new file mode 100644 index 0000000..57473bf --- /dev/null +++ b/repoScaffold/src/platform_cli/commands/install.py @@ -0,0 +1,62 @@ +"""platform-cli install [--check-only]""" +from __future__ import annotations + +import click +from rich.console import Console + +from platform_cli.shell.run import run_cmd +from platform_cli.shell.tools import ( + REQUIRED_TOOLS, + detect_tool, + has_brew, + is_macos, +) + +console = Console() + + +@click.command() +@click.option("--check-only", is_flag=True, help="Only check tools, do not install.") +def install(check_only: bool) -> None: + """Check / install required CLI tools.""" + console.print("\n[bold]Checking required tools...[/bold]\n") + + missing: list[dict[str, str]] = [] + for tool in REQUIRED_TOOLS: + found = detect_tool(tool["cmd"]) + status = "[green]found[/green]" if found else "[red]missing[/red]" + console.print(f" {status} {tool['name']}") + if not found: + missing.append(tool) + + if not missing: + console.print("\n[bold green]All tools found.[/bold green]\n") + return + + if check_only: + console.print("\n[yellow]Missing tools:[/yellow]") + for t in missing: + console.print(f" - {t['name']}: {t['install_hint']}") + return + + if is_macos() and has_brew(): + console.print("\n[bold]Installing missing tools via Homebrew...[/bold]\n") + for t in missing: + brew_pkg = t.get("brew", "") + if not brew_pkg: + console.print(f" [dim]skip[/dim] {t['name']} (bundled with another tool)") + continue + console.print(f" [cyan]install[/cyan] brew install {brew_pkg}") + result = run_cmd(f"brew install {brew_pkg}", timeout=600) + if result.ok: + console.print(f" [green]done[/green] {t['name']}") + else: + console.print(f" [red]failed[/red] {t['name']}: {t['install_hint']}") + console.print() + else: + console.print( + "\n[yellow]Automated install is currently macOS+Homebrew only. " + "Install manually:[/yellow]" + ) + for t in missing: + console.print(f" - {t['name']}: {t['install_hint']}") diff --git a/repoScaffold/src/platform_cli/commands/scaffold.py b/repoScaffold/src/platform_cli/commands/scaffold.py new file mode 100644 index 0000000..015a94c --- /dev/null +++ b/repoScaffold/src/platform_cli/commands/scaffold.py @@ -0,0 +1,96 @@ +"""platform-cli scaffold [--manifest] [--skip-phase] [--dry-run]""" +from __future__ import annotations + +from pathlib import Path + +import click +from rich.console import Console + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import build_dag +from platform_cli.engine.runner import StepRunner +from platform_cli.engine.state import RunState +from platform_cli.manifest.loader import load_manifest +from platform_cli.manifest.schema import ProjectManifest, ProjectConfig, CloudConfig + +# Trigger step registration by importing the steps package +import platform_cli.steps # noqa: F401 + +console = Console() + + +@click.command() +@click.argument("name") +@click.option( + "--manifest", "-m", + type=click.Path(exists=True, path_type=Path), + help="Path to a YAML manifest file.", +) +@click.option( + "--skip-phase", "-s", + multiple=True, + help="Phase letter(s) to skip (e.g. C, G, H).", +) +@click.option("--dry-run", is_flag=True, help="Show execution plan without running.") +@click.option("--no-resume", is_flag=True, help="Ignore previous run state.") +@click.option( + "--output-dir", "-o", + type=click.Path(path_type=Path), + default=None, + help="Parent directory for the generated project (default: cwd).", +) +def scaffold( + name: str, + manifest: Path | None, + skip_phase: tuple[str, ...], + dry_run: bool, + no_resume: bool, + output_dir: Path | None, +) -> None: + """Scaffold a new full-stack project.""" + if manifest: + m = load_manifest(manifest) + # Override project name from CLI arg + m.project.name = name + else: + m = ProjectManifest( + project=ProjectConfig(name=name, cloud=CloudConfig()), + ) + + parent = output_dir or Path.cwd() + project_dir = parent / name + project_dir.mkdir(parents=True, exist_ok=True) + + skip = [p.upper() for p in skip_phase] + + # Library kinds have no services, database, infrastructure, or deploy. + # Auto-skip the web-app build/provision phases (C–M) while keeping repo + # scaffold (B), agent access (N) and Linear (O). The library reuses an + # existing GCP project (cloud.project_id) for Secret Manager, so nothing + # under G is created here. + if m.project.is_library: + LIBRARY_SKIP = ["C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M"] + for p in LIBRARY_SKIP: + if p not in skip: + skip.append(p) + + ctx = ScaffoldContext( + manifest=m, + project_dir=project_dir, + dry_run=dry_run, + skip_phases=skip, + ) + + state_file = project_dir / ".scaffold-state.json" + state = RunState(state_file) + + console.print(f"\n[bold]Scaffolding project:[/bold] {name}") + console.print(f" Directory: {project_dir}") + console.print(f" Skip phases: {list(ctx.skip_phases) or 'none'}") + console.print(f" Dry run: {dry_run}\n") + + steps = build_dag() + runner = StepRunner(steps, state) + runner.run_all(ctx, resume=not no_resume) + + console.print("\n[bold green]Done.[/bold green]\n") diff --git a/repoScaffold/src/platform_cli/commands/test_cmd.py b/repoScaffold/src/platform_cli/commands/test_cmd.py new file mode 100644 index 0000000..9544dc5 --- /dev/null +++ b/repoScaffold/src/platform_cli/commands/test_cmd.py @@ -0,0 +1,117 @@ +"""platform-cli test [service] — run Phase K tests against a scaffolded project.""" +from __future__ import annotations + +import sys +from pathlib import Path + +import click +from rich.console import Console + +console = Console() + + +@click.command("test") +@click.argument("service", required=False, default=None) +@click.option( + "--manifest", "-m", + type=click.Path(exists=True), + default=None, + help="Path to project manifest.", +) +@click.option( + "--project-dir", "-d", + type=click.Path(exists=True), + default=".", + help="Path to the generated project directory.", +) +@click.option( + "--skip", "-s", + multiple=True, + help="Substring(s) matching step_ids to skip (e.g. docker, terraform).", +) +@click.option( + "--only", "-o", + multiple=True, + help="Substring(s) matching step_ids to include (overrides service filter).", +) +def test( + service: str | None, + manifest: str | None, + project_dir: str, + skip: tuple[str, ...], + only: tuple[str, ...], +) -> None: + """Run tests against a scaffolded project. + + Without arguments, runs all Phase K tests. Pass a service name (api, web, + worker) to run only its tests, or use --only/--skip for finer control. + """ + from platform_cli.engine.context import ScaffoldContext + from platform_cli.engine.registry import build_dag + from platform_cli.engine.state import RunState + from platform_cli.manifest.loader import load_manifest + from platform_cli.manifest.schema import ProjectManifest, ProjectConfig, CloudConfig + import platform_cli.steps # noqa: F401 + + pdir = Path(project_dir) + manifest_path = Path(manifest) if manifest else pdir / "project.manifest.yaml" + + if manifest_path.exists(): + m = load_manifest(manifest_path) + else: + m = ProjectManifest(project=ProjectConfig(name=pdir.name, cloud=CloudConfig())) + + ctx = ScaffoldContext(manifest=m, project_dir=pdir) + + all_steps = build_dag() + test_steps = [s for s in all_steps if s.phase == "K"] + + if only: + needles = [o.lower() for o in only] + test_steps = [s for s in test_steps if any(n in s.step_id.lower() for n in needles)] + elif service: + svc_lower = service.lower() + test_steps = [s for s in test_steps if svc_lower in s.step_id.lower()] + + if skip: + needles = [s.lower() for s in skip] + test_steps = [s for s in test_steps if not any(n in s.step_id.lower() for n in needles)] + + if not test_steps: + console.print("[yellow]No matching test steps found.[/yellow]") + return + + state = RunState(pdir / ".test-state.json") + state.clear() + + console.print(f"\n[bold]Running tests ({len(test_steps)} steps)...[/bold]\n") + + passed: list[str] = [] + failed: list[tuple[str, str]] = [] + skipped: list[str] = [] + + for step in test_steps: + if step.should_skip(ctx): + console.print(f" [yellow]skip[/yellow] {step.step_id}") + skipped.append(step.step_id) + continue + try: + console.print(f" [green]run[/green] {step.step_id}") + step.run(ctx) + console.print(f" [green]pass[/green] {step.step_id}") + passed.append(step.step_id) + except Exception as exc: # noqa: BLE001 + msg = str(exc).splitlines()[0][:200] if str(exc) else exc.__class__.__name__ + console.print(f" [red]fail[/red] {step.step_id}: {msg}") + failed.append((step.step_id, str(exc))) + + console.print("") + console.print(f"[bold]Summary:[/bold] {len(passed)} passed, {len(failed)} failed, {len(skipped)} skipped") + for sid, err in failed: + console.print(f" [red]FAIL[/red] {sid}") + for line in err.splitlines()[:6]: + console.print(f" {line}") + + if failed: + sys.exit(1) + console.print("\n[bold green]All tests passed.[/bold green]\n") diff --git a/repoScaffold/src/platform_cli/engine/__init__.py b/repoScaffold/src/platform_cli/engine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/repoScaffold/src/platform_cli/engine/context.py b/repoScaffold/src/platform_cli/engine/context.py new file mode 100644 index 0000000..d952582 --- /dev/null +++ b/repoScaffold/src/platform_cli/engine/context.py @@ -0,0 +1,49 @@ +"""ScaffoldContext: shared state bag passed to every step.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from platform_cli.manifest.schema import ProjectManifest + + +@dataclass +class ScaffoldContext: + """Mutable state bag shared across all steps during a scaffold run.""" + + manifest: ProjectManifest + project_dir: Path + dry_run: bool = False + skip_phases: list[str] = field(default_factory=list) + extras: dict[str, Any] = field(default_factory=dict) + + @property + def project_name(self) -> str: + return self.manifest.project.name + + @property + def project_id(self) -> str: + return self.manifest.project.cloud.project_id + + @property + def region(self) -> str: + return self.manifest.project.cloud.region + + @property + def org_id(self) -> str: + """Org id pinned in the manifest (empty = auto-detect).""" + return getattr(self.manifest.project.cloud, "org_id", "") or "" + + def service(self, svc_type: str): + """Return the first enabled service matching *svc_type*, or None.""" + for s in self.manifest.services: + if s.type == svc_type and s.enabled: + return s + return None + + def set(self, key: str, value: Any) -> None: + self.extras[key] = value + + def get(self, key: str, default: Any = None) -> Any: + return self.extras.get(key, default) diff --git a/repoScaffold/src/platform_cli/engine/dag.py b/repoScaffold/src/platform_cli/engine/dag.py new file mode 100644 index 0000000..c8a0fa9 --- /dev/null +++ b/repoScaffold/src/platform_cli/engine/dag.py @@ -0,0 +1,44 @@ +"""DAG builder + Kahn's topological sort.""" +from __future__ import annotations + +from collections import defaultdict, deque + +from platform_cli.engine.step import BaseStep + + +class CycleError(Exception): + """Raised when the step DAG contains a cycle.""" + + +def topological_sort(steps: list[BaseStep]) -> list[BaseStep]: + """Return steps in dependency-respecting order using Kahn's algorithm.""" + step_map: dict[str, BaseStep] = {s.step_id: s for s in steps} + + # Build adjacency and in-degree + in_degree: dict[str, int] = defaultdict(int) + dependents: dict[str, list[str]] = defaultdict(list) + + for s in steps: + in_degree.setdefault(s.step_id, 0) + for dep in s.depends_on: + dependents[dep].append(s.step_id) + in_degree[s.step_id] += 1 + + queue: deque[str] = deque( + sid for sid, deg in in_degree.items() if deg == 0 + ) + ordered: list[str] = [] + + while queue: + sid = queue.popleft() + ordered.append(sid) + for child in dependents[sid]: + in_degree[child] -= 1 + if in_degree[child] == 0: + queue.append(child) + + if len(ordered) != len(steps): + remaining = set(s.step_id for s in steps) - set(ordered) + raise CycleError(f"Cycle detected involving steps: {remaining}") + + return [step_map[sid] for sid in ordered] diff --git a/repoScaffold/src/platform_cli/engine/registry.py b/repoScaffold/src/platform_cli/engine/registry.py new file mode 100644 index 0000000..3069fb9 --- /dev/null +++ b/repoScaffold/src/platform_cli/engine/registry.py @@ -0,0 +1,24 @@ +"""@register_step decorator + build_dag() helper.""" +from __future__ import annotations + +from platform_cli.engine.dag import topological_sort +from platform_cli.engine.step import BaseStep + +_REGISTRY: list[type[BaseStep]] = [] + + +def register_step(cls: type[BaseStep]) -> type[BaseStep]: + """Class decorator that registers a step for DAG construction.""" + _REGISTRY.append(cls) + return cls + + +def build_dag() -> list[BaseStep]: + """Instantiate all registered steps and return them in topological order.""" + steps = [cls() for cls in _REGISTRY] + return topological_sort(steps) + + +def registered_steps() -> list[type[BaseStep]]: + """Return the raw list of registered step classes.""" + return list(_REGISTRY) diff --git a/repoScaffold/src/platform_cli/engine/runner.py b/repoScaffold/src/platform_cli/engine/runner.py new file mode 100644 index 0000000..a4d3b6d --- /dev/null +++ b/repoScaffold/src/platform_cli/engine/runner.py @@ -0,0 +1,72 @@ +"""StepRunner: execute steps with retry + state persistence.""" +from __future__ import annotations + +import time +from typing import Any + +from rich.console import Console +from rich.panel import Panel + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.state import RunState +from platform_cli.engine.step import BaseStep + +console = Console() + + +class StepRunner: + """Execute an ordered list of steps, persisting progress.""" + + def __init__(self, steps: list[BaseStep], state: RunState) -> None: + self.steps = steps + self.state = state + + def run_all(self, ctx: ScaffoldContext, *, resume: bool = True) -> None: + if not resume: + self.state.clear() + + for step in self.steps: + if resume and self.state.is_completed(step.step_id): + console.print(f" [dim]skip (done)[/dim] {step.step_id}") + continue + + if step.should_skip(ctx): + console.print(f" [yellow]skip (phase)[/yellow] {step.step_id}") + continue + + if ctx.dry_run: + console.print(f" [cyan]dry-run[/cyan] {step.step_id}") + continue + + self._execute(step, ctx) + + def _execute(self, step: BaseStep, ctx: ScaffoldContext) -> dict[str, Any]: + attempts = step.max_retries + 1 + last_err: Exception | None = None + + for attempt in range(1, attempts + 1): + try: + console.print(f" [green]run[/green] {step.step_id}", end="") + if attempts > 1: + console.print(f" [dim](attempt {attempt}/{attempts})[/dim]", end="") + console.print() + + outputs = step.run(ctx) or {} + self.state.mark_completed(step.step_id, outputs) + return outputs + except Exception as exc: + last_err = exc + if attempt < attempts: + console.print(f" [yellow]retry[/yellow] {step.step_id}: {exc}") + time.sleep(1) + + assert last_err is not None + self.state.mark_failed(step.step_id, str(last_err)) + console.print( + Panel( + f"[red bold]Step failed:[/red bold] {step.step_id}\n{last_err}", + title="Error", + border_style="red", + ) + ) + raise last_err diff --git a/repoScaffold/src/platform_cli/engine/state.py b/repoScaffold/src/platform_cli/engine/state.py new file mode 100644 index 0000000..a1d33e4 --- /dev/null +++ b/repoScaffold/src/platform_cli/engine/state.py @@ -0,0 +1,42 @@ +"""RunState: JSON persistence for resumability.""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +class RunState: + """Tracks which steps have completed and their outputs. + + State is persisted as JSON so a failed run can resume where it left off. + """ + + def __init__(self, state_file: Path) -> None: + self._path = state_file + self._data: dict[str, Any] = {"completed": {}, "failed": None} + if self._path.exists(): + self._data = json.loads(self._path.read_text()) + + def is_completed(self, step_id: str) -> bool: + return step_id in self._data["completed"] + + def mark_completed(self, step_id: str, outputs: dict[str, Any]) -> None: + self._data["completed"][step_id] = outputs + self._save() + + def mark_failed(self, step_id: str, error: str) -> None: + self._data["failed"] = {"step_id": step_id, "error": error} + self._save() + + def outputs(self, step_id: str) -> dict[str, Any]: + return self._data["completed"].get(step_id, {}) + + def clear(self) -> None: + self._data = {"completed": {}, "failed": None} + if self._path.exists(): + self._path.unlink() + + def _save(self) -> None: + self._path.parent.mkdir(parents=True, exist_ok=True) + self._path.write_text(json.dumps(self._data, indent=2)) diff --git a/repoScaffold/src/platform_cli/engine/step.py b/repoScaffold/src/platform_cli/engine/step.py new file mode 100644 index 0000000..2fd7cb6 --- /dev/null +++ b/repoScaffold/src/platform_cli/engine/step.py @@ -0,0 +1,38 @@ +"""BaseStep ABC: the contract every scaffold step implements.""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any + +from platform_cli.engine.context import ScaffoldContext + + +class BaseStep(ABC): + """Abstract base for every scaffold step. + + Subclasses must set the class-level attributes and implement ``run()``. + """ + + step_id: str = "" + phase: str = "" + depends_on: list[str] = [] + max_retries: int = 0 + + @abstractmethod + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + """Execute the step, returning a dict of outputs.""" + + def inputs(self) -> list[str]: + """Descriptive list of what this step needs (for documentation).""" + return [] + + def outputs_spec(self) -> list[str]: + """Descriptive list of what this step produces.""" + return [] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + """Return True if this step should be skipped given the context.""" + return self.phase in ctx.skip_phases + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} id={self.step_id}>" diff --git a/repoScaffold/src/platform_cli/manifest/__init__.py b/repoScaffold/src/platform_cli/manifest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/repoScaffold/src/platform_cli/manifest/defaults.py b/repoScaffold/src/platform_cli/manifest/defaults.py new file mode 100644 index 0000000..a4ddfae --- /dev/null +++ b/repoScaffold/src/platform_cli/manifest/defaults.py @@ -0,0 +1,18 @@ +"""Default values applied when the manifest omits fields.""" + +DEFAULT_PORTS: dict[str, int] = { + "api": 3006, + "webapp": 3005, + "worker": 8080, +} + +DEFAULT_STACKS: dict[str, str] = { + "api": "express", + "webapp": "react", + "worker": "python", +} + +# Default agent service account — automatically added to every project's +# agent access list. This is the shared agent VM identity. +# Change this if the agent machine is recreated. +DEFAULT_AGENT_SERVICE_ACCOUNT = "464961297779-compute@developer.gserviceaccount.com" diff --git a/repoScaffold/src/platform_cli/manifest/loader.py b/repoScaffold/src/platform_cli/manifest/loader.py new file mode 100644 index 0000000..1c58419 --- /dev/null +++ b/repoScaffold/src/platform_cli/manifest/loader.py @@ -0,0 +1,37 @@ +"""Load YAML manifest, validate with Pydantic, and apply defaults.""" +from __future__ import annotations + +from pathlib import Path + +import yaml + +from platform_cli.manifest.defaults import DEFAULT_PORTS, DEFAULT_STACKS +from platform_cli.manifest.schema import ProjectManifest + + +def load_manifest(path: Path) -> ProjectManifest: + """Read a YAML manifest, apply defaults, return a validated model.""" + raw = yaml.safe_load(path.read_text()) + manifest = ProjectManifest.model_validate(raw) + _apply_defaults(manifest) + return manifest + + +def _apply_defaults(m: ProjectManifest) -> None: + name_lower = m.project.name.lower().replace(" ", "-") + + if not m.project.cloud.project_id: + m.project.cloud.project_id = name_lower + + if not m.database.db_name: + m.database.db_name = name_lower.replace("-", "_") + + for svc in m.services: + if not svc.name: + svc.name = svc.type + if not svc.subdomain: + svc.subdomain = svc.name + if svc.port == 0: + svc.port = DEFAULT_PORTS.get(svc.type, 8080) + if not svc.stack: + svc.stack = DEFAULT_STACKS.get(svc.type, "") diff --git a/repoScaffold/src/platform_cli/manifest/schema.py b/repoScaffold/src/platform_cli/manifest/schema.py new file mode 100644 index 0000000..17c9926 --- /dev/null +++ b/repoScaffold/src/platform_cli/manifest/schema.py @@ -0,0 +1,97 @@ +"""Pydantic models for the project manifest.""" +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class CloudConfig(BaseModel): + provider: str = "gcp" + region: str = "us-central1" + project_id: str = "" + # Organization to place the GCP project under. Empty = auto-detect the + # caller's org (see platform_cli.steps._gcp_org). Pin it here to force a + # specific org, or override at runtime with PLATFORM_GCP_ORG_ID. + org_id: str = "" + + +class ProjectConfig(BaseModel): + # Project kind drives which phases run and which templates render. + # app — full-stack web app (services + infra + deploy). Default. + # library-node — publishable Node/npm package. No services, DB, infra, + # or deploy; the agent builds/tests/publishes locally. + # Namespaced so future variants (library-python, library-react, …) slot in. + kind: str = "app" + name: str + env: str = "dev" + # GitHub repo as "owner/name" — used to wire the Cloud Build trigger to an + # existing repo (libraries publish from a repo that already exists). + repo: str = "" + cloud: CloudConfig = Field(default_factory=CloudConfig) + + @property + def is_library(self) -> bool: + return self.kind.startswith("library") + + +class DatabaseConfig(BaseModel): + type: str = "mongodb" + atlas_cluster: str = "Cluster0" + db_name: str = "" + + +class ServiceConfig(BaseModel): + type: str # api | webapp | worker + enabled: bool = True + name: str = "" + subdomain: str = "" + stack: str = "" + port: int = 0 + gpu: str = "none" + # Worker-only: cron schedule (Cloud Scheduler format) + # Default: every 10 minutes + schedule: str = "*/10 * * * *" + # Worker-only: max parallel task instances per scheduled run + parallelism: int = 1 + # Worker-only: max execution time per task (seconds) + timeout_seconds: int = 600 + + +class AgentRoleConfig(BaseModel): + name: str # e.g. "developer", "reviewer", "ops" + sleep_seconds: int = 600 # seconds between iterations + max_turns: int = 1000 # max Claude turns per iteration + + +class AgentConfig(BaseModel): + service_accounts: list[str] = Field(default_factory=list) + roles: list[AgentRoleConfig] = Field(default_factory=list) + + +class LibraryConfig(BaseModel): + """Settings for `kind: library-*` projects (publishable packages). + + Keeps the library agent templates generic: the package may live at the + repo root or a subdirectory, and build/test/publish commands vary by + toolchain (tsup, rollup, vite, etc.). + """ + package_dir: str = "." # dir holding package.json, relative to repo root + build_cmd: str = "npm run build" + test_cmd: str = "npm test" + publish_cmd: str = "npm publish" + registry_url: str = "https://www.npmjs.com/package" # for README/board links + + +class LinearConfig(BaseModel): + enabled: bool = False + workspace: str = "" # e.g. "ai4us" + team_key: str = "" # e.g. "AI4" — the prefix for issue IDs + team_name: str = "" # e.g. "AI4US Team" + + +class ProjectManifest(BaseModel): + project: ProjectConfig + database: DatabaseConfig = Field(default_factory=DatabaseConfig) + services: list[ServiceConfig] = Field(default_factory=list) + library: LibraryConfig = Field(default_factory=LibraryConfig) + agents: AgentConfig = Field(default_factory=AgentConfig) + linear: LinearConfig = Field(default_factory=LinearConfig) diff --git a/repoScaffold/src/platform_cli/shell/__init__.py b/repoScaffold/src/platform_cli/shell/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/repoScaffold/src/platform_cli/shell/run.py b/repoScaffold/src/platform_cli/shell/run.py new file mode 100644 index 0000000..8240a5f --- /dev/null +++ b/repoScaffold/src/platform_cli/shell/run.py @@ -0,0 +1,57 @@ +"""subprocess wrapper: ShellResult and run_cmd.""" +from __future__ import annotations + +import subprocess +from dataclasses import dataclass + + +@dataclass +class ShellResult: + command: str + returncode: int + stdout: str + stderr: str + + @property + def ok(self) -> bool: + return self.returncode == 0 + + +def run_cmd( + cmd: str | list[str], + *, + cwd: str | None = None, + check: bool = False, + capture: bool = True, + timeout: int = 300, +) -> ShellResult: + """Run a shell command and return a ShellResult.""" + if isinstance(cmd, str): + shell = True + cmd_str = cmd + else: + shell = False + cmd_str = " ".join(cmd) + + result = subprocess.run( + cmd, + shell=shell, + cwd=cwd, + capture_output=capture, + text=True, + timeout=timeout, + ) + + sr = ShellResult( + command=cmd_str, + returncode=result.returncode, + stdout=result.stdout if capture else "", + stderr=result.stderr if capture else "", + ) + + if check and not sr.ok: + raise RuntimeError( + f"Command failed (rc={sr.returncode}): {cmd_str}\n{sr.stderr}" + ) + + return sr diff --git a/repoScaffold/src/platform_cli/shell/tools.py b/repoScaffold/src/platform_cli/shell/tools.py new file mode 100644 index 0000000..b1e3c1e --- /dev/null +++ b/repoScaffold/src/platform_cli/shell/tools.py @@ -0,0 +1,63 @@ +"""Tool detection and install hints.""" +from __future__ import annotations + +import platform +import shutil + +REQUIRED_TOOLS: list[dict[str, str]] = [ + { + "name": "Google Cloud SDK", + "cmd": "gcloud", + "brew": "google-cloud-sdk", + "install_hint": "brew install --cask google-cloud-sdk (or https://cloud.google.com/sdk/docs/install)", + }, + { + "name": "Terraform", + "cmd": "terraform", + "brew": "terraform", + "install_hint": "brew install terraform (or https://developer.hashicorp.com/terraform/install)", + }, + { + "name": "Docker", + "cmd": "docker", + "brew": "--cask docker", + "install_hint": "brew install --cask docker (or https://docs.docker.com/get-docker/)", + }, + { + "name": "Node.js", + "cmd": "node", + "brew": "node", + "install_hint": "brew install node (or https://nodejs.org/)", + }, + { + "name": "npm", + "cmd": "npm", + "brew": "", + "install_hint": "Installed with Node.js", + }, + { + "name": "MongoDB Atlas CLI", + "cmd": "atlas", + "brew": "mongodb-atlas-cli", + "install_hint": "brew install mongodb-atlas-cli (or https://www.mongodb.com/docs/atlas/cli/current/install-atlas-cli/)", + }, + { + "name": "GitHub CLI", + "cmd": "gh", + "brew": "gh", + "install_hint": "brew install gh (or https://cli.github.com/)", + }, +] + + +def detect_tool(cmd: str) -> bool: + """Return True if *cmd* is found on PATH.""" + return shutil.which(cmd) is not None + + +def is_macos() -> bool: + return platform.system() == "Darwin" + + +def has_brew() -> bool: + return shutil.which("brew") is not None diff --git a/repoScaffold/src/platform_cli/steps/__init__.py b/repoScaffold/src/platform_cli/steps/__init__.py new file mode 100644 index 0000000..0419d8b --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/__init__.py @@ -0,0 +1,19 @@ +"""Import all phase modules to trigger @register_step decorators.""" +from platform_cli.steps import ( # noqa: F401 + phase_a_tools, + phase_b_scaffold, + phase_c_database, + phase_d_api, + phase_e_frontend, + phase_f_worker, + phase_g_gcp, + phase_h_secrets, + phase_i_cloudbuild, + phase_j_terraform, + phase_k_testing, + phase_l_deploy, + phase_m_pipelines, + phase_n_agents, + phase_o_linear, + phase_p_library_pipeline, +) diff --git a/repoScaffold/src/platform_cli/steps/_gcp_org.py b/repoScaffold/src/platform_cli/steps/_gcp_org.py new file mode 100644 index 0000000..90ef5ad --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/_gcp_org.py @@ -0,0 +1,88 @@ +"""Shared GCP organization helpers. + +Every project the CLI creates (or reuses) should land under the caller's +organization when one exists. A project created without an org parent ends up +under "No organization" — invisible in the org-scoped console project picker +and outside org policies/IAM. That is exactly how `fresler-table` slipped out +(org auto-detection returned empty at create time and creation silently +proceeded orgless). + +Two pieces: + * ``resolve_org_id`` — decide which org new projects go under. + * ``ensure_project_in_org`` — adopt an already-created orphan project into + that org (self-healing, so a flaky detection at create time is corrected). +""" +from __future__ import annotations + +import os + +from rich.console import Console + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.shell.run import run_cmd + +console = Console() + +# Runtime override, highest precedence. Handy in CI or one-off runs. +ORG_ENV_VAR = "PLATFORM_GCP_ORG_ID" + + +def resolve_org_id(ctx: ScaffoldContext) -> str: + """Return the org id to place new projects under, or "" if none. + + Resolution order: ``PLATFORM_GCP_ORG_ID`` env var → manifest + ``project.cloud.org_id`` → auto-detect via ``gcloud organizations list``. + """ + env_org = os.environ.get(ORG_ENV_VAR, "").strip() + if env_org: + return env_org + + manifest_org = (ctx.org_id or "").strip() + if manifest_org: + return manifest_org + + listing = run_cmd("gcloud organizations list --format='value(ID)'") + if listing.ok and listing.stdout.strip(): + orgs = [o for o in listing.stdout.strip().split("\n") if o.strip()] + if len(orgs) > 1: + console.print( + f" [yellow]note[/yellow] multiple orgs found; using {orgs[0]} " + f"(set {ORG_ENV_VAR} or project.cloud.org_id to choose)" + ) + if orgs: + return orgs[0] + + console.print( + " [yellow]note[/yellow] no GCP organization detected; project will live " + "under 'No organization'" + ) + return "" + + +def ensure_project_in_org(project_id: str, org_id: str) -> None: + """Adopt an orphaned project into ``org_id``. + + Only moves a project that currently has *no* org parent, so we never yank a + project out of a different org it deliberately belongs to. No-op when + ``org_id`` is empty or the project is already under an organization. + """ + if not org_id: + return + parent = run_cmd( + f"gcloud projects describe {project_id} --format='value(parent.type)'" + ) + if not parent.ok: + return + if parent.stdout.strip() == "organization": + return # already under an org — leave it alone + + moved = run_cmd( + f"gcloud beta projects move {project_id} --organization {org_id} --quiet" + ) + if moved.ok: + console.print(f" [green]moved[/green] {project_id} under org {org_id}") + else: + console.print( + f" [yellow]warn[/yellow] could not move {project_id} under org " + f"{org_id}: {moved.stderr.strip()}" + ) diff --git a/repoScaffold/src/platform_cli/steps/phase_a_tools.py b/repoScaffold/src/platform_cli/steps/phase_a_tools.py new file mode 100644 index 0000000..6975547 --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_a_tools.py @@ -0,0 +1,238 @@ +"""Phase A: Tool detection, installation, and authentication. + + A.1 detect_tools — check PATH for required CLIs + A.2 install_tools — list missing tools with install hints + A.3 authenticate — run all logins sequentially (gcloud, atlas, gh, linear) +""" +from __future__ import annotations + +import os +import subprocess +from typing import Any + +from rich.console import Console + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.shell.run import run_cmd +from platform_cli.shell.tools import REQUIRED_TOOLS, detect_tool + +console = Console() + + +@register_step +class DetectToolsStep(BaseStep): + step_id = "A.1_detect_tools" + phase = "A" + depends_on: list[str] = [] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + results: dict[str, bool] = {} + for tool in REQUIRED_TOOLS: + results[tool["cmd"]] = detect_tool(tool["cmd"]) + ctx.set("detected_tools", results) + return {"detected_tools": results} + + +@register_step +class InstallToolsStep(BaseStep): + step_id = "A.2_install_tools" + phase = "A" + depends_on = ["A.1_detect_tools"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + detected = ctx.get("detected_tools", {}) + missing = [ + t for t in REQUIRED_TOOLS if not detected.get(t["cmd"], False) + ] + if missing: + console.print("[yellow]Missing tools (install manually):[/yellow]") + for t in missing: + console.print(f" - {t['name']}: {t['install_hint']}") + return {"missing_tools": [t["cmd"] for t in missing]} + + +def _get_linear_api_key() -> str: + """Retrieve the Linear API key from ~/.linear/ config or env var.""" + # Check env var first + key = os.environ.get("LINEAR_API_KEY", "") + if key: + return key + + # Check ~/.linear/ config (stored by @linear/cli) + config_path = os.path.expanduser("~/.linear/config.json") + if os.path.exists(config_path): + import json + try: + with open(config_path) as f: + config = json.load(f) + key = config.get("apiKey", "") + if key: + return key + except (json.JSONDecodeError, KeyError): + pass + + return "" + + +def _is_logged_in(cmd: str, check_cmd: str) -> bool: + """Check if a CLI tool is authenticated.""" + r = run_cmd(check_cmd, timeout=15) + return r.ok + + +def _interactive_login(label: str, login_cmd: str) -> bool: + """Run an interactive login command, letting it use the real terminal.""" + console.print(f" [cyan]logging in[/cyan] {label}...") + try: + result = subprocess.run( + login_cmd, + shell=True, + timeout=300, + ) + return result.returncode == 0 + except subprocess.TimeoutExpired: + console.print(f" [yellow]timeout[/yellow] waiting for {label} login") + return False + + +@register_step +class AuthenticateStep(BaseStep): + """Consolidated login step — authenticates all required CLIs sequentially. + + Checks each tool's auth status first and only prompts for login when + needed. Interactive logins inherit the real terminal so browser-based + OAuth flows work correctly. + """ + + step_id = "A.3_authenticate" + phase = "A" + depends_on = ["A.2_install_tools"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + detected = ctx.get("detected_tools", {}) + auth_status: dict[str, str] = {} + + # ── Google Cloud ── + if detected.get("gcloud"): + if _is_logged_in("gcloud", "gcloud config get-value account"): + who = run_cmd("gcloud config get-value account") + auth_status["gcloud"] = who.stdout.strip() + console.print(f" [dim]gcloud[/dim] → {auth_status['gcloud']}") + else: + if _interactive_login("Google Cloud", "gcloud auth login"): + auth_status["gcloud"] = "ok" + console.print(" [green]gcloud[/green] → authenticated") + else: + raise RuntimeError( + "gcloud login failed. Run `gcloud auth login` manually." + ) + + # Also ensure application-default credentials for Terraform + adc = run_cmd( + "gcloud auth application-default print-access-token", + timeout=10, + ) + if not adc.ok: + console.print( + " [cyan]setting up[/cyan] application-default credentials (for Terraform)..." + ) + _interactive_login( + "GCP ADC", + "gcloud auth application-default login", + ) + + # ── MongoDB Atlas ── + if detected.get("atlas"): + if _is_logged_in("atlas", "atlas auth whoami"): + who = run_cmd("atlas auth whoami") + auth_status["atlas"] = who.stdout.strip() + console.print(f" [dim]atlas[/dim] → {auth_status['atlas']}") + else: + if _interactive_login("MongoDB Atlas", "atlas auth login"): + auth_status["atlas"] = "ok" + console.print(" [green]atlas[/green] → authenticated") + else: + raise RuntimeError( + "Atlas login failed. Run `atlas auth login` manually." + ) + + # ── GitHub CLI ── + if detected.get("gh"): + if _is_logged_in("gh", "gh auth status"): + who = run_cmd("gh api user --jq '.login'") + auth_status["gh"] = who.stdout.strip() if who.ok else "ok" + console.print(f" [dim]gh[/dim] → {auth_status['gh']}") + else: + if _interactive_login("GitHub", "gh auth login"): + auth_status["gh"] = "ok" + console.print(" [green]gh[/green] → authenticated") + else: + raise RuntimeError( + "GitHub login failed. Run `gh auth login` manually." + ) + + # ── Linear ── + linear_cfg = ctx.manifest.linear + if linear_cfg.enabled: + api_key = _get_linear_api_key() + if api_key: + # Verify the key works + r = run_cmd( + f"curl -sf -H 'Authorization: {api_key}' " + f"-H 'Content-Type: application/json' " + f"-d '{{\"query\": \"{{ viewer {{ id name }} }}\"}}' " + f"https://api.linear.app/graphql", + timeout=15, + ) + if r.ok and "viewer" in r.stdout: + auth_status["linear"] = "ok" + console.print(" [dim]linear[/dim] → authenticated") + else: + console.print(" [yellow]linear[/yellow] stored key is invalid") + api_key = "" + + if not api_key: + # Prompt the user to enter their API key + console.print( + " [yellow]linear[/yellow] not authenticated" + ) + console.print( + " Create an API key at: https://linear.app/settings/api" + ) + console.print( + " Then paste it here or set LINEAR_API_KEY env var and re-run." + ) + try: + key_input = input(" Linear API key: ").strip() + if key_input: + # Store it + config_dir = os.path.expanduser("~/.linear") + os.makedirs(config_dir, exist_ok=True) + import json as _json + config_path = os.path.join(config_dir, "config.json") + _json.dump({"apiKey": key_input}, open(config_path, "w")) + os.chmod(config_path, 0o600) + auth_status["linear"] = "ok" + console.print(" [green]linear[/green] → key saved") + else: + auth_status["linear"] = "not_authenticated" + except (EOFError, KeyboardInterrupt): + auth_status["linear"] = "not_authenticated" + + # ── Docker ── + if detected.get("docker"): + r = run_cmd("docker info", timeout=10) + if r.ok: + auth_status["docker"] = "running" + console.print(" [dim]docker[/dim] → running") + else: + console.print(" [cyan]starting[/cyan] Docker Desktop...") + run_cmd("open -a Docker") + # Don't block — Docker takes time to start. + # Phase L will wait for it when needed. + auth_status["docker"] = "starting" + + ctx.set("auth_status", auth_status) + return {"auth_status": auth_status} diff --git a/repoScaffold/src/platform_cli/steps/phase_b_scaffold.py b/repoScaffold/src/platform_cli/steps/phase_b_scaffold.py new file mode 100644 index 0000000..aec08ef --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_b_scaffold.py @@ -0,0 +1,323 @@ +"""Phase B: Repository / folder scaffold steps.""" +from __future__ import annotations + +from typing import Any + +import yaml + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.templates.renderer import render_to_file, _TEMPLATES_DIR + + +def _kind_template(base: str, ext: str, ctx: ScaffoldContext) -> str: + """Pick a kind-specific template variant if one exists, else the base. + + e.g. base="project/AGENTS", ext="md.j2" resolves to + "project/AGENTS.library-node.md.j2" for a library-node project when that + file exists, otherwise falls back to "project/AGENTS.md.j2". + """ + if ctx.manifest.project.is_library: + variant = f"{base}.{ctx.manifest.project.kind}.{ext}" + if (_TEMPLATES_DIR / variant).exists(): + return variant + return f"{base}.{ext}" + + +@register_step +class CreateDirectories(BaseStep): + step_id = "B.1_create_directories" + phase = "B" + depends_on = ["A.2_install_tools"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + if ctx.manifest.project.is_library: + # A library has no services/infra dirs, but does get a cloudbuild/ + # dir for its npm build+publish pipeline. + dirs = ["agents", ".claude", "cloudbuild"] + else: + dirs = [ + "cli", + "services/api/src", + "services/web", + "services/worker", + "infra/terraform/modules", + "infra/terraform/envs/dev", + "cloudbuild", + "cloudrun", + "agents", + ".vscode", + ".claude", + ] + for d in dirs: + (ctx.project_dir / d).mkdir(parents=True, exist_ok=True) + return {"directories_created": len(dirs)} + + +@register_step +class WriteManifest(BaseStep): + step_id = "B.2_write_manifest" + phase = "B" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + dest = ctx.project_dir / "project.manifest.yaml" + data = ctx.manifest.model_dump() + dest.write_text(yaml.dump(data, default_flow_style=False, sort_keys=False)) + return {"manifest_path": str(dest)} + + +@register_step +class WriteGitignore(BaseStep): + step_id = "B.3_write_gitignore" + phase = "B" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + render_to_file( + "project/.gitignore.j2", + ctx.project_dir / ".gitignore", + project=ctx.manifest.project, + ) + return {} + + +@register_step +class WriteDockerCompose(BaseStep): + step_id = "B.4_write_docker_compose" + phase = "B" + depends_on = ["B.1_create_directories"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + # Libraries have no services to compose. + return super().should_skip(ctx) or ctx.manifest.project.is_library + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + render_to_file( + "project/docker-compose.yml.j2", + ctx.project_dir / "docker-compose.yml", + manifest=ctx.manifest, + services=ctx.manifest.services, + ) + return {} + + +@register_step +class WriteAgentsMd(BaseStep): + step_id = "B.5_write_agents_md" + phase = "B" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + render_to_file( + _kind_template("project/AGENTS", "md.j2", ctx), + ctx.project_dir / "agents" / "AGENTS.md", + manifest=ctx.manifest, + ) + return {} + + +@register_step +class WriteVSCode(BaseStep): + step_id = "B.6_write_vscode" + phase = "B" + depends_on = ["B.1_create_directories"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + # The launch/tasks configs target services + ports — not relevant to a library. + return super().should_skip(ctx) or ctx.manifest.project.is_library + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + render_to_file( + "vscode/launch.json.j2", + ctx.project_dir / ".vscode" / "launch.json", + manifest=ctx.manifest, + ) + render_to_file( + "vscode/tasks.json.j2", + ctx.project_dir / ".vscode" / "tasks.json", + manifest=ctx.manifest, + ) + return {} + + +@register_step +class WriteClaude(BaseStep): + step_id = "B.7_write_claude" + phase = "B" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + render_to_file( + _kind_template("claude/settings", "json.j2", ctx), + ctx.project_dir / ".claude" / "settings.json", + manifest=ctx.manifest, + ) + return {} + + +@register_step +class WriteReadme(BaseStep): + step_id = "B.8_write_readme" + phase = "B" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + render_to_file( + _kind_template("project/README", "md.j2", ctx), + ctx.project_dir / "README.md", + manifest=ctx.manifest, + ) + return {} + + +@register_step +class WriteEnvExample(BaseStep): + step_id = "B.9_write_env_example" + phase = "B" + depends_on = ["B.1_create_directories"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + # The .env.example lists service/DB secrets — not relevant to a library. + return super().should_skip(ctx) or ctx.manifest.project.is_library + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + render_to_file( + "project/.env.example.j2", + ctx.project_dir / ".env.example", + manifest=ctx.manifest, + ) + return {} + + +@register_step +class WriteLibraryCloudbuild(BaseStep): + """Render the npm build+publish Cloud Build pipeline for library projects.""" + + step_id = "B.11_write_library_cloudbuild" + phase = "B" + depends_on = ["B.1_create_directories"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not ctx.manifest.project.is_library + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + render_to_file( + "cloudbuild/npm-publish.yaml.j2", + ctx.project_dir / "cloudbuild" / "publish.yaml", + manifest=ctx.manifest, + ) + return {"cloudbuild": "cloudbuild/publish.yaml"} + + +@register_step +class WriteAgentLoop(BaseStep): + step_id = "B.10_write_agent_loop" + phase = "B" + depends_on = ["B.1_create_directories"] + + # Built-in agent templates shipped with the scaffolder. + # A library defaults to just the developer agent (the reviewer/ops agents + # monitor Cloud Run and are meaningless without a deployed service). + BUILTIN_AGENTS = ["developer", "reviewer", "ops"] + BUILTIN_AGENTS_LIBRARY = ["developer"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + import os + from pathlib import Path + + # Determine which agents to scaffold + manifest_roles = {r.name: r for r in ctx.manifest.agents.roles} + default_agents = ( + self.BUILTIN_AGENTS_LIBRARY + if ctx.manifest.project.is_library + else self.BUILTIN_AGENTS + ) + agent_names = list(manifest_roles.keys()) if manifest_roles else default_agents + + agents_dir = ctx.project_dir / "agents" + scaffolded = [] + + for name in agent_names: + agent_dir = agents_dir / name + agent_dir.mkdir(parents=True, exist_ok=True) + + role = manifest_roles.get(name) + agent_ctx = { + "sleep_seconds": role.sleep_seconds if role else None, + "max_turns": role.max_turns if role else None, + } + + template_dir = Path(f"agents/{name}") + prompt_template = _kind_template(f"agents/{name}/PROMPT", "md.j2", ctx) + config_template = f"agents/{name}/config.yaml.j2" + + try: + render_to_file( + prompt_template, + agent_dir / "PROMPT.md", + manifest=ctx.manifest, + agent=agent_ctx, + ) + except Exception: + # No built-in template — create a placeholder + (agent_dir / "PROMPT.md").write_text( + f"# {name.title()} Agent\n\n" + f"You are the **{name}** agent for **{ctx.manifest.project.name}**.\n\n" + f"Define this agent's behavior by editing this file.\n" + ) + + try: + render_to_file( + config_template, + agent_dir / "config.yaml", + manifest=ctx.manifest, + agent=agent_ctx, + ) + except Exception: + defaults = {"developer": (600, 1000), "reviewer": (1800, 200), "ops": (1800, 100)} + sleep_s, max_t = defaults.get(name, (600, 500)) + (agent_dir / "config.yaml").write_text( + f"name: {name}\n" + f"description: {name.title()} agent\n" + f"sleep_seconds: {agent_ctx.get('sleep_seconds') or sleep_s}\n" + f"max_turns: {agent_ctx.get('max_turns') or max_t}\n" + ) + + scaffolded.append(name) + + # Write shared loop.sh and setup-vm.sh into agents/ + render_to_file( + "project/loop.sh.j2", + agents_dir / "loop.sh", + manifest=ctx.manifest, + ) + render_to_file( + "project/setup-vm.sh.j2", + agents_dir / "setup-vm.sh", + manifest=ctx.manifest, + ) + # Deterministic board-reconciliation helper (no Claude); loop.sh runs it + # for one designated agent each cycle. Takes the team key as an argument. + render_to_file( + "project/reconcile.py.j2", + agents_dir / "reconcile.py", + manifest=ctx.manifest, + ) + # Uploads a screenshot to an unlisted gist and prints a markdown image tag + # that renders INLINE in a PR body — the only reliable way to attach an + # image to a private-repo PR (see the script header). Used by developers + # for the required web/UI screenshot. + render_to_file( + "project/pr-screenshot.sh.j2", + agents_dir / "pr-screenshot.sh", + manifest=ctx.manifest, + ) + + os.chmod(agents_dir / "loop.sh", 0o755) + os.chmod(agents_dir / "setup-vm.sh", 0o755) + os.chmod(agents_dir / "reconcile.py", 0o755) + os.chmod(agents_dir / "pr-screenshot.sh", 0o755) + + return {"agents": scaffolded} diff --git a/repoScaffold/src/platform_cli/steps/phase_c_database.py b/repoScaffold/src/platform_cli/steps/phase_c_database.py new file mode 100644 index 0000000..56a559a --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_c_database.py @@ -0,0 +1,191 @@ +"""Phase C: Database setup (MongoDB Atlas). + +Idempotent flow: + C.1 atlas_auth — ensure logged in + C.2 ensure_cluster — verify cluster exists + C.3 ensure_db_user — create DB user + strong password (if not existing) + C.4 get_connection_string — resolve the SRV URI and bake user credentials in + C.5 seed_items — create `items` collection and upsert a sample doc + C.6 write_api_env — write MONGODB_URI + DB_NAME to services/api/.env +""" +from __future__ import annotations + +import json +import secrets +import shlex +import string +from typing import Any +from urllib.parse import quote_plus + +from rich.console import Console + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.shell.run import run_cmd + +console = Console() + + +def _gen_password(n: int = 24) -> str: + alphabet = string.ascii_letters + string.digits + return "".join(secrets.choice(alphabet) for _ in range(n)) + + +@register_step +class AtlasAuth(BaseStep): + step_id = "C.1_atlas_auth" + phase = "C" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + if run_cmd("atlas auth whoami").ok: + return {"atlas_authenticated": True} + raise RuntimeError( + "Not logged in to Atlas. Run `atlas auth login` " + "(or `atlas config init` with an API key) and re-run." + ) + + +@register_step +class EnsureCluster(BaseStep): + step_id = "C.2_ensure_cluster" + phase = "C" + depends_on = ["C.1_atlas_auth"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + cluster = ctx.manifest.database.atlas_cluster + result = run_cmd(f"atlas clusters describe {cluster} -o json") + if not result.ok: + raise RuntimeError( + f"Atlas cluster '{cluster}' not found in the current project. " + f"Either create it in the Atlas UI, or update " + f"`database.atlas_cluster` in the manifest.\n{result.stderr}" + ) + data = json.loads(result.stdout) + ctx.set("atlas_cluster_data", data) + return {"cluster": cluster, "state": data.get("stateName")} + + +@register_step +class EnsureDbUser(BaseStep): + step_id = "C.3_ensure_db_user" + phase = "C" + depends_on = ["C.2_ensure_cluster"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + username = (ctx.project_name.lower().replace(" ", "-") + "-app")[:64] + existing = run_cmd(f"atlas dbusers describe {username} -o json") + if existing.ok: + # User exists — rotate password so we can bake it into the URI. + password = _gen_password() + run_cmd( + f"atlas dbusers update {username} --password {shlex.quote(password)}", + check=True, + ) + else: + password = _gen_password() + run_cmd( + f"atlas dbusers create atlasAdmin " + f"--username {username} " + f"--password {shlex.quote(password)}", + check=True, + ) + ctx.set("db_username", username) + ctx.set("db_password", password) + return {"username": username} + + +@register_step +class GetConnectionString(BaseStep): + step_id = "C.4_get_connection_string" + phase = "C" + depends_on = ["C.3_ensure_db_user"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + cluster = ctx.manifest.database.atlas_cluster + db_name = ctx.manifest.database.db_name + + r = run_cmd(f"atlas clusters connectionStrings describe {cluster} -o json") + if not r.ok: + raise RuntimeError(f"Failed to get Atlas connection string: {r.stderr}") + data = json.loads(r.stdout) + srv = data.get("standardSrv") or data.get("standard") + if not srv: + raise RuntimeError(f"No connection string found: {data}") + + user = quote_plus(ctx.get("db_username", "")) + pw = quote_plus(ctx.get("db_password", "")) + host = srv.split("://", 1)[1] + uri = f"mongodb+srv://{user}:{pw}@{host}/{db_name}?retryWrites=true&w=majority" + ctx.set("mongodb_uri", uri) + return {"has_uri": True} + + +@register_step +class SeedItems(BaseStep): + step_id = "C.5_seed_items" + phase = "C" + depends_on = ["C.4_get_connection_string"] + max_retries = 2 + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + uri = ctx.get("mongodb_uri") + db_name = ctx.manifest.database.db_name + script = ( + "db.items.updateOne(" + '{"name":"example item"},' + '{"$setOnInsert":{"name":"example item"}},' + "{upsert:true});" + 'print("ok");' + ) + cmd = f"mongosh {shlex.quote(uri)} --quiet --eval {shlex.quote(script)}" + r = run_cmd(cmd, timeout=60) + if r.ok and "ok" in r.stdout: + return {"seeded": True, "db": db_name} + + # Fallback: mongosh bundles its own Node runtime whose c-ares + # resolver can fail SRV lookups. Use the system `node` with + # the mongodb driver (installed via the API service) instead. + if "ETIMEOUT" in (r.stderr or r.stdout): + api_dir = ctx.project_dir / "services" / "api" + node_script = ( + "const {MongoClient}=require('mongodb');" + f"const c=new MongoClient({shlex.quote(uri)});" + "async function main(){await c.connect();" + f"const db=c.db({shlex.quote(db_name)});" + "await db.collection('items').updateOne(" + '{"name":"example item"},' + '{"$setOnInsert":{"name":"example item"}},' + "{upsert:true});" + 'console.log("ok");await c.close();}' + "main().catch(e=>{console.error(e.message);process.exit(1);});" + ) + fb = run_cmd( + f"node -e {shlex.quote(node_script)}", + cwd=str(api_dir), + timeout=60, + ) + if fb.ok and "ok" in fb.stdout: + return {"seeded": True, "db": db_name, "fallback": "node"} + raise RuntimeError(f"Seed failed (node fallback): {fb.stderr or fb.stdout}") + + raise RuntimeError(f"Seed failed: {r.stderr or r.stdout}") + + +@register_step +class WriteApiEnv(BaseStep): + step_id = "C.6_write_api_env" + phase = "C" + depends_on = ["C.4_get_connection_string"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + uri = ctx.get("mongodb_uri") + db_name = ctx.manifest.database.db_name + env_file = ctx.project_dir / "services" / "api" / ".env" + env_file.parent.mkdir(parents=True, exist_ok=True) + env_file.write_text( + f"MONGODB_URI={uri}\n" + f"DB_NAME={db_name}\n" + ) + return {"env_written": str(env_file)} diff --git a/repoScaffold/src/platform_cli/steps/phase_d_api.py b/repoScaffold/src/platform_cli/steps/phase_d_api.py new file mode 100644 index 0000000..83327cb --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_d_api.py @@ -0,0 +1,134 @@ +"""Phase D: API scaffold steps (Express + Mongoose).""" +from __future__ import annotations + +from typing import Any + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.shell.run import run_cmd +from platform_cli.templates.renderer import render_to_file + + +def _api_enabled(ctx: ScaffoldContext) -> bool: + return ctx.service("api") is not None + + +@register_step +class InitExpress(BaseStep): + step_id = "D.1_init_express" + phase = "D" + depends_on = ["B.1_create_directories"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _api_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + api_dir = ctx.project_dir / "services" / "api" + api_dir.mkdir(parents=True, exist_ok=True) + + svc = ctx.service("api") + render_to_file( + "services/api/package.json.j2", + api_dir / "package.json", + project=ctx.manifest.project, + service=svc, + ) + return {"api_package_json": True} + + +@register_step +class AddMongoose(BaseStep): + step_id = "D.2_add_mongoose" + phase = "D" + depends_on = ["D.1_init_express"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _api_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + # Mongoose is included in the package.json template + return {"mongoose_configured": True} + + +@register_step +class WriteApiSource(BaseStep): + step_id = "D.3_write_api_source" + phase = "D" + depends_on = ["D.2_add_mongoose"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _api_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + api_src = ctx.project_dir / "services" / "api" / "src" + api_src.mkdir(parents=True, exist_ok=True) + + svc = ctx.service("api") + db = ctx.manifest.database + + render_to_file( + "services/api/index.js.j2", + api_src / "index.js", + service=svc, + database=db, + ) + render_to_file( + "services/api/db.js.j2", + api_src / "db.js", + database=db, + ) + render_to_file( + "services/api/models.js.j2", + api_src / "models.js", + database=db, + ) + render_to_file( + "services/api/.eslintrc.json.j2", + ctx.project_dir / "services" / "api" / ".eslintrc.json", + ) + return {"api_source_written": True} + + +@register_step +class WriteApiDockerfile(BaseStep): + step_id = "D.4_write_api_dockerfile" + phase = "D" + depends_on = ["D.1_init_express"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _api_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + api_dir = ctx.project_dir / "services" / "api" + render_to_file( + "services/api/Dockerfile.j2", + api_dir / "Dockerfile", + ) + render_to_file( + "services/api/.dockerignore.j2", + api_dir / ".dockerignore", + ) + return {} + + +@register_step +class InitializeApiEnv(BaseStep): + step_id = "D.5_write_api_env" + phase = "D" + depends_on = ["D.1_init_express"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _api_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + # PORT is NOT written here — local runs use the manifest default + # from index.js, and Docker sets PORT=80 via its own ENV. + env_file = ctx.project_dir / "services" / "api" / ".env" + if not env_file.exists(): + db = ctx.manifest.database + env_file.write_text( + f"MONGODB_URI=mongodb+srv://:@{db.atlas_cluster.lower()}.xxxxx.mongodb.net/{db.db_name}?retryWrites=true&w=majority\n" + f"DB_NAME={db.db_name}\n" + ) + return {} diff --git a/repoScaffold/src/platform_cli/steps/phase_e_frontend.py b/repoScaffold/src/platform_cli/steps/phase_e_frontend.py new file mode 100644 index 0000000..45501b8 --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_e_frontend.py @@ -0,0 +1,151 @@ +"""Phase E: Frontend scaffold steps (React).""" +from __future__ import annotations + +from typing import Any + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.shell.run import run_cmd +from platform_cli.templates.renderer import render_to_file + + +def _web_enabled(ctx: ScaffoldContext) -> bool: + return ctx.service("webapp") is not None + + +@register_step +class InitReact(BaseStep): + step_id = "E.1_init_react" + phase = "E" + depends_on = ["B.1_create_directories"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _web_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + web_dir = ctx.project_dir / "services" / "web" + package_json = web_dir / "package.json" + + if package_json.exists(): + return {"react_initialized": True, "source": "existing"} + + # Vite is non-interactive when --template is provided. + result = run_cmd( + f"npm create vite@latest {web_dir.name} -- --template react", + cwd=str(web_dir.parent), + timeout=120, + ) + if result.ok and package_json.exists(): + return {"react_initialized": True, "source": "vite"} + + # Fallback: render our template set directly. + svc = ctx.service("webapp") + api_svc = ctx.service("api") + api_port = api_svc.port if api_svc else 3006 + web_port = svc.port if svc else 3005 + for tmpl, rel in [ + ("services/web/package.json.j2", "package.json"), + ("services/web/vite.config.js.j2", "vite.config.js"), + ("services/web/index.html.j2", "index.html"), + ("services/web/main.jsx.j2", "src/main.jsx"), + ]: + render_to_file( + tmpl, + web_dir / rel, + project=ctx.manifest.project, + service=svc, + api_port=api_port, + web_port=web_port, + ) + return {"react_initialized": True, "source": "fallback"} + + +@register_step +class ConfigureProxy(BaseStep): + step_id = "E.2_configure_proxy" + phase = "E" + depends_on = ["E.1_init_react"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _web_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + web_dir = ctx.project_dir / "services" / "web" + + svc = ctx.service("webapp") + api_svc = ctx.service("api") + api_port = api_svc.port if api_svc else 3006 + web_port = svc.port if svc else 3005 + + render_to_file( + "services/web/vite.config.js.j2", + web_dir / "vite.config.js", + api_port=api_port, + web_port=web_port, + ) + return {"proxy_configured": True, "api_port": api_port, "web_port": web_port} + + +@register_step +class WriteItemsFetch(BaseStep): + step_id = "E.3_write_items_fetch" + phase = "E" + depends_on = ["E.2_configure_proxy"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _web_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + web_src = ctx.project_dir / "services" / "web" / "src" + web_src.mkdir(parents=True, exist_ok=True) + + api_svc = ctx.service("api") + api_port = api_svc.port if api_svc else 3006 + + render_to_file( + "services/web/App.jsx.j2", + web_src / "App.jsx", + api_port=api_port, + ) + render_to_file( + "services/web/api.js.j2", + web_src / "api.js", + api_port=api_port, + ) + render_to_file( + "services/web/index.html.j2", + ctx.project_dir / "services" / "web" / "index.html", + project=ctx.manifest.project, + ) + # Drop the CRA-era App.js if Vite didn't overwrite it (Vite's default is App.jsx). + legacy_app = web_src / "App.js" + if legacy_app.exists(): + legacy_app.unlink() + return {"items_fetch_written": True} + + +@register_step +class WriteFrontendDockerfile(BaseStep): + step_id = "E.4_write_frontend_dockerfile" + phase = "E" + depends_on = ["E.1_init_react"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _web_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + api_svc = ctx.service("api") + api_name = api_svc.name if api_svc else "api" + web_dir = ctx.project_dir / "services" / "web" + + render_to_file( + "services/web/Dockerfile.j2", + web_dir / "Dockerfile", + api_name=api_name, + ) + render_to_file( + "services/web/.dockerignore.j2", + web_dir / ".dockerignore", + ) + return {} diff --git a/repoScaffold/src/platform_cli/steps/phase_f_worker.py b/repoScaffold/src/platform_cli/steps/phase_f_worker.py new file mode 100644 index 0000000..e1ab5a0 --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_f_worker.py @@ -0,0 +1,77 @@ +"""Phase F: Worker scaffold steps.""" +from __future__ import annotations + +from typing import Any + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.templates.renderer import render_to_file + + +def _worker_enabled(ctx: ScaffoldContext) -> bool: + return ctx.service("worker") is not None + + +@register_step +class ScaffoldWorker(BaseStep): + step_id = "F.1_scaffold_worker" + phase = "F" + depends_on = ["B.1_create_directories"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _worker_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + worker_dir = ctx.project_dir / "services" / "worker" + worker_dir.mkdir(parents=True, exist_ok=True) + + svc = ctx.service("worker") + + render_to_file( + "services/worker/main.py.j2", + worker_dir / "main.py", + service=svc, + ) + render_to_file( + "services/worker/requirements.txt.j2", + worker_dir / "requirements.txt", + service=svc, + ) + return {"worker_scaffolded": True} + + +@register_step +class WriteWorkerDockerfile(BaseStep): + step_id = "F.2_write_worker_dockerfile" + phase = "F" + depends_on = ["F.1_scaffold_worker"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _worker_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + svc = ctx.service("worker") + + render_to_file( + "services/worker/Dockerfile.j2", + ctx.project_dir / "services" / "worker" / "Dockerfile", + service=svc, + ) + return {} + + +@register_step +class WriteWorkerEnv(BaseStep): + step_id = "F.3_write_worker_env" + phase = "F" + depends_on = ["F.1_scaffold_worker"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not _worker_enabled(ctx) + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + env_file = ctx.project_dir / "services" / "worker" / ".env" + if not env_file.exists(): + env_file.write_text("PORT=80\n") + return {} diff --git a/repoScaffold/src/platform_cli/steps/phase_g_gcp.py b/repoScaffold/src/platform_cli/steps/phase_g_gcp.py new file mode 100644 index 0000000..ff20b59 --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_g_gcp.py @@ -0,0 +1,215 @@ +"""Phase G: GCP project setup steps.""" +from __future__ import annotations + +from typing import Any + +from rich.console import Console + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.shell.run import run_cmd +from platform_cli.steps._gcp_org import ensure_project_in_org, resolve_org_id + +console = Console() + + +@register_step +class CreateGcpProject(BaseStep): + step_id = "G.1_create_gcp_project" + phase = "G" + depends_on = ["B.1_create_directories"] + max_retries = 0 + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + + # Require gcloud auth + who = run_cmd("gcloud config get-value account") + if not who.ok or not who.stdout.strip(): + raise RuntimeError( + "No gcloud account found. Run `gcloud auth login` then retry." + ) + + # Which org should this project live under? (env → manifest → detect) + org_id = resolve_org_id(ctx) + + # Reuse an existing project — still make sure it's under the org, so an + # orphaned (no-org) project gets adopted rather than left invisible. + if run_cmd(f"gcloud projects describe {project_id}").ok: + run_cmd(f"gcloud config set project {project_id}", check=True) + ensure_project_in_org(project_id, org_id) + return {"gcp_project_exists": True, "project_id": project_id} + + # Build the create command + create_cmd = f"gcloud projects create {project_id} --name={ctx.project_name}" + if org_id: + create_cmd += f" --organization={org_id}" + + result = run_cmd(create_cmd) + if not result.ok: + raise RuntimeError( + f"Could not create GCP project '{project_id}'. " + "This usually means you need a billing account + organization, " + "or the project ID is taken. Either (a) set `project.cloud.project_id` " + "in the manifest to an existing project you own, or (b) create the " + "project manually in the GCP console and re-run with --skip-phase (no G).\n" + f"gcloud error: {result.stderr}" + ) + run_cmd(f"gcloud config set project {project_id}", check=True) + # Belt-and-suspenders: if --organization was dropped for any reason, + # adopt the fresh project into the org now. + ensure_project_in_org(project_id, org_id) + + # Auto-link billing account so subsequent API enables work without + # manual intervention. Pick the first open billing account. + billing_list = run_cmd( + "gcloud billing accounts list --filter='OPEN=true' --format='value(ACCOUNT_ID)' --limit=1" + ) + billing_id = billing_list.stdout.strip().split("\n")[0] if billing_list.ok else "" + if billing_id: + link = run_cmd( + f"gcloud billing projects link {project_id} --billing-account={billing_id}" + ) + if link.ok: + console.print(f" [green]linked billing[/green] {billing_id}") + + return { + "gcp_project_created": True, + "project_id": project_id, + "org_id": org_id, + "billing_account": billing_id, + } + + +@register_step +class EnableApis(BaseStep): + step_id = "G.2_enable_apis" + phase = "G" + depends_on = ["G.1_create_gcp_project"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + apis = [ + "run.googleapis.com", + "cloudbuild.googleapis.com", + "secretmanager.googleapis.com", + "artifactregistry.googleapis.com", + "compute.googleapis.com", + "vpcaccess.googleapis.com", + "cloudscheduler.googleapis.com", + ] + project_id = ctx.project_id + + for api in apis: + run_cmd( + f"gcloud services enable {api} --project={project_id}", + check=True, + ) + return {"apis_enabled": apis} + + +@register_step +class CreateArtifactRegistry(BaseStep): + step_id = "G.3_create_artifact_registry" + phase = "G" + depends_on = ["G.2_enable_apis"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + region = ctx.region + repo_name = f"{ctx.project_name.lower().replace(' ', '-')}-docker" + + # Check if registry exists + result = run_cmd( + f"gcloud artifacts repositories describe {repo_name} " + f"--location={region} --project={project_id}" + ) + if result.ok: + return {"registry_exists": True, "repo_name": repo_name} + + run_cmd( + f"gcloud artifacts repositories create {repo_name} " + f"--repository-format=docker " + f"--location={region} " + f"--project={project_id}", + check=True, + ) + ctx.set("artifact_registry", repo_name) + return {"registry_created": True, "repo_name": repo_name} + + +@register_step +class CreateServiceAccounts(BaseStep): + step_id = "G.4_create_service_accounts" + phase = "G" + depends_on = ["G.1_create_gcp_project"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + prefix = ctx.project_name.lower().replace(" ", "-") + + accounts = { + "build": f"{prefix}-build", + "runtime": f"{prefix}-runtime", + } + + for role, sa_name in accounts.items(): + email = f"{sa_name}@{project_id}.iam.gserviceaccount.com" + result = run_cmd( + f"gcloud iam service-accounts describe {email} " + f"--project={project_id}" + ) + if not result.ok: + run_cmd( + f"gcloud iam service-accounts create {sa_name} " + f'--display-name="{ctx.project_name} {role} SA" ' + f"--project={project_id}", + check=True, + ) + + ctx.set("service_accounts", accounts) + return {"service_accounts": accounts} + + +@register_step +class ConfigureIam(BaseStep): + step_id = "G.5_configure_iam" + phase = "G" + depends_on = ["G.4_create_service_accounts"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + accounts = ctx.get("service_accounts", {}) + prefix = ctx.project_name.lower().replace(" ", "-") + + build_email = f"{accounts.get('build', prefix + '-build')}@{project_id}.iam.gserviceaccount.com" + runtime_email = f"{accounts.get('runtime', prefix + '-runtime')}@{project_id}.iam.gserviceaccount.com" + + # Build SA roles + build_roles = [ + "roles/cloudbuild.builds.builder", + "roles/artifactregistry.writer", + "roles/run.admin", + "roles/secretmanager.secretAccessor", + ] + for role in build_roles: + run_cmd( + f"gcloud projects add-iam-policy-binding {project_id} " + f"--member=serviceAccount:{build_email} " + f"--role={role} --quiet" + ) + + # Runtime SA roles + runtime_roles = [ + "roles/secretmanager.secretAccessor", + "roles/logging.logWriter", + "roles/monitoring.metricWriter", + ] + for role in runtime_roles: + run_cmd( + f"gcloud projects add-iam-policy-binding {project_id} " + f"--member=serviceAccount:{runtime_email} " + f"--role={role} --quiet" + ) + + return {"iam_configured": True} diff --git a/repoScaffold/src/platform_cli/steps/phase_h_secrets.py b/repoScaffold/src/platform_cli/steps/phase_h_secrets.py new file mode 100644 index 0000000..f2a5931 --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_h_secrets.py @@ -0,0 +1,119 @@ +"""Phase H: Secret management steps.""" +from __future__ import annotations + +import tempfile +from pathlib import Path +from typing import Any + +import yaml + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.shell.run import run_cmd + + +def _read_env(path: Path) -> dict[str, str]: + out: dict[str, str] = {} + if not path.exists(): + return out + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, val = line.partition("=") + out[key.strip()] = val.strip() + return out + + +@register_step +class WriteSecretManifest(BaseStep): + step_id = "H.1_write_secret_manifest" + phase = "H" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + secrets = [ + {"name": "MONGODB_URI", "source": "services/api/.env"}, + {"name": "DB_NAME", "source": "services/api/.env"}, + # Agent VM secrets — provisioned directly to Secret Manager (never + # via .env), read by the agent loop at runtime. Listed here for + # discoverability; created by their respective phases. + {"name": "claude-code-oauth-token", "source": "env:CLAUDE_CODE_OAUTH_TOKEN (agent Claude auth)"}, + {"name": "linear-api-key", "source": "env:LINEAR_API_KEY (agent Linear access)"}, + {"name": "github-deploy-key", "source": "generated (agent git auth)"}, + ] + + manifest_path = ctx.project_dir / "secrets.manifest.yaml" + manifest_path.write_text( + yaml.dump( + {"secrets": secrets}, + default_flow_style=False, + sort_keys=False, + ) + ) + return {"secret_manifest_path": str(manifest_path)} + + +@register_step +class SyncSecretsToGcp(BaseStep): + step_id = "H.2_sync_secrets_to_gcp" + phase = "H" + depends_on = ["H.1_write_secret_manifest", "G.2_enable_apis"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + env_vars = _read_env(ctx.project_dir / "services" / "api" / ".env") + + synced: list[str] = [] + for key, value in env_vars.items(): + secret_name = key.lower().replace("_", "-") + + exists = run_cmd( + f"gcloud secrets describe {secret_name} --project={project_id}" + ).ok + if not exists: + run_cmd( + f"gcloud secrets create {secret_name} " + f"--project={project_id} --replication-policy=automatic", + check=True, + ) + + # Write value to a temp file to avoid shell-escaping pitfalls. + with tempfile.NamedTemporaryFile( + "w", delete=False, prefix="secret-", suffix=".txt" + ) as tmp: + tmp.write(value) + tmp_path = tmp.name + try: + run_cmd( + f"gcloud secrets versions add {secret_name} " + f"--data-file={tmp_path} --project={project_id}", + check=True, + ) + finally: + Path(tmp_path).unlink(missing_ok=True) + + synced.append(key) + + return {"synced_secrets": synced} + + +@register_step +class WriteEnvExampleSecrets(BaseStep): + step_id = "H.3_write_env_example_secrets" + phase = "H" + depends_on = ["H.1_write_secret_manifest"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + env_example = ctx.project_dir / ".env.example" + if env_example.exists(): + content = env_example.read_text() + if "# Secrets (managed by GCP Secret Manager)" not in content: + content += ( + "\n# Secrets (managed by GCP Secret Manager)\n" + "# MONGODB_URI=\n" + "# DB_NAME=\n" + ) + env_example.write_text(content) + return {} diff --git a/repoScaffold/src/platform_cli/steps/phase_i_cloudbuild.py b/repoScaffold/src/platform_cli/steps/phase_i_cloudbuild.py new file mode 100644 index 0000000..f331cf0 --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_i_cloudbuild.py @@ -0,0 +1,98 @@ +"""Phase I: Cloud Build config steps.""" +from __future__ import annotations + +from typing import Any + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.templates.renderer import render_to_file + + +@register_step +class WriteApiBuild(BaseStep): + step_id = "I.1_write_api_build" + phase = "I" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + svc = ctx.service("api") + if not svc: + return {"skipped": True} + + render_to_file( + "cloudbuild/api.yaml.j2", + ctx.project_dir / "cloudbuild" / "api.yaml", + manifest=ctx.manifest, + service=svc, + ) + + render_to_file( + "cloudrun/api.yaml.j2", + ctx.project_dir / "cloudrun" / "api.yaml", + manifest=ctx.manifest, + service=svc, + ) + return {} + + +@register_step +class WriteWebBuild(BaseStep): + step_id = "I.2_write_web_build" + phase = "I" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + svc = ctx.service("webapp") + if not svc: + return {"skipped": True} + + render_to_file( + "cloudbuild/web.yaml.j2", + ctx.project_dir / "cloudbuild" / "web.yaml", + manifest=ctx.manifest, + service=svc, + ) + + render_to_file( + "cloudrun/web.yaml.j2", + ctx.project_dir / "cloudrun" / "web.yaml", + manifest=ctx.manifest, + service=svc, + ) + return {} + + +@register_step +class WriteWorkerBuild(BaseStep): + step_id = "I.3_write_worker_build" + phase = "I" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + svc = ctx.service("worker") + if not svc: + return {"skipped": True} + + render_to_file( + "cloudbuild/worker.yaml.j2", + ctx.project_dir / "cloudbuild" / "worker.yaml", + manifest=ctx.manifest, + service=svc, + ) + return {} + + +@register_step +class WriteTerraformBuild(BaseStep): + step_id = "I.4_write_terraform_build" + phase = "I" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + render_to_file( + "cloudbuild/terraform.yaml.j2", + ctx.project_dir / "cloudbuild" / "terraform.yaml", + manifest=ctx.manifest, + ) + return {} diff --git a/repoScaffold/src/platform_cli/steps/phase_j_terraform.py b/repoScaffold/src/platform_cli/steps/phase_j_terraform.py new file mode 100644 index 0000000..fd5084b --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_j_terraform.py @@ -0,0 +1,164 @@ +"""Phase J: Terraform generation steps.""" +from __future__ import annotations + +from typing import Any + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.templates.renderer import render_to_file + + +@register_step +class WriteProviderTf(BaseStep): + step_id = "J.1_write_provider_tf" + phase = "J" + depends_on = ["B.1_create_directories"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + render_to_file( + "infra/terraform/main.tf.j2", + ctx.project_dir / "infra" / "terraform" / "main.tf", + manifest=ctx.manifest, + ) + render_to_file( + "infra/terraform/variables.tf.j2", + ctx.project_dir / "infra" / "terraform" / "variables.tf", + manifest=ctx.manifest, + ) + render_to_file( + "infra/terraform/outputs.tf.j2", + ctx.project_dir / "infra" / "terraform" / "outputs.tf", + manifest=ctx.manifest, + ) + return {} + + +@register_step +class WriteCloudRunModule(BaseStep): + step_id = "J.2_write_cloudrun_module" + phase = "J" + depends_on = ["J.1_write_provider_tf"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + mod_dir = "infra/terraform/modules/cloudrun" + + render_to_file( + f"{mod_dir}/main.tf.j2", + ctx.project_dir / mod_dir / "main.tf", + manifest=ctx.manifest, + ) + render_to_file( + f"{mod_dir}/variables.tf.j2", + ctx.project_dir / mod_dir / "variables.tf", + manifest=ctx.manifest, + ) + render_to_file( + f"{mod_dir}/outputs.tf.j2", + ctx.project_dir / mod_dir / "outputs.tf", + manifest=ctx.manifest, + ) + return {} + + +@register_step +class WriteIamModule(BaseStep): + step_id = "J.3_write_iam_module" + phase = "J" + depends_on = ["J.1_write_provider_tf"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + mod_dir = "infra/terraform/modules/iam" + + render_to_file( + f"{mod_dir}/main.tf.j2", + ctx.project_dir / mod_dir / "main.tf", + manifest=ctx.manifest, + ) + render_to_file( + f"{mod_dir}/variables.tf.j2", + ctx.project_dir / mod_dir / "variables.tf", + manifest=ctx.manifest, + ) + render_to_file( + f"{mod_dir}/outputs.tf.j2", + ctx.project_dir / mod_dir / "outputs.tf", + manifest=ctx.manifest, + ) + return {} + + +@register_step +class WriteSecretsModule(BaseStep): + step_id = "J.4_write_secrets_module" + phase = "J" + depends_on = ["J.1_write_provider_tf"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + mod_dir = "infra/terraform/modules/secrets" + + render_to_file( + f"{mod_dir}/main.tf.j2", + ctx.project_dir / mod_dir / "main.tf", + manifest=ctx.manifest, + ) + render_to_file( + f"{mod_dir}/variables.tf.j2", + ctx.project_dir / mod_dir / "variables.tf", + manifest=ctx.manifest, + ) + return {} + + +@register_step +class WriteNetworkingModule(BaseStep): + step_id = "J.5_write_networking_module" + phase = "J" + depends_on = ["J.1_write_provider_tf"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + mod_dir = "infra/terraform/modules/networking" + + render_to_file( + f"{mod_dir}/main.tf.j2", + ctx.project_dir / mod_dir / "main.tf", + manifest=ctx.manifest, + ) + render_to_file( + f"{mod_dir}/variables.tf.j2", + ctx.project_dir / mod_dir / "variables.tf", + manifest=ctx.manifest, + ) + return {} + + +@register_step +class WriteDevEnv(BaseStep): + step_id = "J.6_write_dev_env" + phase = "J" + depends_on = [ + "J.2_write_cloudrun_module", + "J.3_write_iam_module", + "J.4_write_secrets_module", + "J.5_write_networking_module", + ] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + env_dir = "infra/terraform/envs/dev" + + render_to_file( + f"{env_dir}/main.tf.j2", + ctx.project_dir / env_dir / "main.tf", + manifest=ctx.manifest, + ) + render_to_file( + f"{env_dir}/variables.tf.j2", + ctx.project_dir / env_dir / "variables.tf", + manifest=ctx.manifest, + ) + render_to_file( + f"{env_dir}/terraform.tfvars.j2", + ctx.project_dir / env_dir / "terraform.tfvars", + manifest=ctx.manifest, + ) + return {} diff --git a/repoScaffold/src/platform_cli/steps/phase_k_testing.py b/repoScaffold/src/platform_cli/steps/phase_k_testing.py new file mode 100644 index 0000000..d6fccd1 --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_k_testing.py @@ -0,0 +1,185 @@ +"""Phase K: Testing steps.""" +from __future__ import annotations + +import os +import signal +import subprocess +import time +from typing import Any + +from rich.console import Console + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.shell.run import run_cmd + +console = Console() + + +def _ensure_deps(dir_path: str) -> None: + if not os.path.isdir(os.path.join(dir_path, "node_modules")): + run_cmd("npm install", cwd=dir_path, check=True, timeout=300) + + +@register_step +class ApiLint(BaseStep): + step_id = "K.1_api_lint" + phase = "K" + depends_on = ["D.3_write_api_source"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or ctx.service("api") is None + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + api_dir = str(ctx.project_dir / "services" / "api") + _ensure_deps(api_dir) + result = run_cmd("npm run lint", cwd=api_dir) + if not result.ok: + raise RuntimeError(f"Lint failed:\n{result.stdout}\n{result.stderr}") + return {"lint_passed": True} + + +@register_step +class ApiHealth(BaseStep): + step_id = "K.2_api_health" + phase = "K" + depends_on = ["D.3_write_api_source"] + max_retries = 0 + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or ctx.service("api") is None + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + svc = ctx.service("api") + port = svc.port if svc else 3006 + api_dir = str(ctx.project_dir / "services" / "api") + _ensure_deps(api_dir) + + # Spawn the API in its own process group so we can kill the whole tree. + proc = subprocess.Popen( + ["node", "src/index.js"], + cwd=api_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + preexec_fn=os.setsid, + ) + + health_ok = False + last_err = "" + try: + deadline = time.time() + 20 + url = f"http://localhost:{port}/health" + while time.time() < deadline: + r = run_cmd(f"curl -sf {url}", timeout=5) + if r.ok: + health_ok = True + break + last_err = r.stderr or r.stdout + time.sleep(0.5) + finally: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + except ProcessLookupError: + pass + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + + if not health_ok: + logs = "" + if proc.stdout: + logs = proc.stdout.read() or "" + raise RuntimeError( + f"API /health did not respond on :{port}\nlast curl: {last_err}\nprocess log:\n{logs[-1000:]}" + ) + return {"health_ok": True, "port": port} + + +@register_step +class ApiDockerBuild(BaseStep): + step_id = "K.3_api_docker_build" + phase = "K" + depends_on = ["D.4_write_api_dockerfile"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or ctx.service("api") is None + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + api_dir = str(ctx.project_dir / "services" / "api") + tag = f"{ctx.project_name.lower().replace(' ', '-')}-api:test" + result = run_cmd( + f"docker build -t {tag} .", + cwd=api_dir, + check=True, + timeout=600, + ) + return {"image_tag": tag, "build_ok": result.ok} + + +@register_step +class FrontendSmoke(BaseStep): + step_id = "K.4_frontend_smoke" + phase = "K" + depends_on = ["E.3_write_items_fetch"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or ctx.service("webapp") is None + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + web_dir = str(ctx.project_dir / "services" / "web") + _ensure_deps(web_dir) + result = run_cmd("npm run build", cwd=web_dir, timeout=300) + if not result.ok: + raise RuntimeError( + f"Frontend build failed:\n{result.stdout}\n{result.stderr}" + ) + return {"build_ok": True} + + +@register_step +class TerraformValidate(BaseStep): + step_id = "K.5_terraform_validate" + phase = "K" + depends_on = ["J.6_write_dev_env"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + tf_dir = str(ctx.project_dir / "infra" / "terraform" / "envs" / "dev") + run_cmd("terraform init -backend=false", cwd=tf_dir, check=True, timeout=180) + result = run_cmd("terraform validate", cwd=tf_dir) + if not result.ok: + raise RuntimeError( + f"terraform validate failed:\n{result.stdout}\n{result.stderr}" + ) + return {"validate_ok": True} + + +@register_step +class TerraformPlan(BaseStep): + """Plan requires GCP credentials; soft-fail with a clear warning.""" + + step_id = "K.6_terraform_plan" + phase = "K" + depends_on = ["K.5_terraform_validate"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + import subprocess + + tf_dir = str(ctx.project_dir / "infra" / "terraform" / "envs" / "dev") + try: + result = run_cmd("terraform plan -input=false", cwd=tf_dir, timeout=300) + except subprocess.TimeoutExpired: + console.print( + "[yellow]terraform plan timed out — soft-fail. " + "Phase L will run apply which exercises the same provider auth.[/yellow]" + ) + return {"plan_ok": False, "soft_fail": "timeout"} + + if not result.ok: + console.print( + "[yellow]terraform plan did not succeed — usually needs GCP auth " + "and the project to exist. Run `gcloud auth application-default login`.[/yellow]" + ) + return {"plan_ok": result.ok} diff --git a/repoScaffold/src/platform_cli/steps/phase_l_deploy.py b/repoScaffold/src/platform_cli/steps/phase_l_deploy.py new file mode 100644 index 0000000..8468c8c --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_l_deploy.py @@ -0,0 +1,273 @@ +"""Phase L: Initial deployment — build+push images, terraform apply, verify live. + + L.1 docker_build_push — build and push images for each service to Artifact Registry + L.2 terraform_apply — apply infrastructure (Cloud Run, IAM, secrets, VPC) + L.3 verify_deployment — hit each service URL and confirm 2xx +""" +from __future__ import annotations + +import json +import time +from typing import Any + +from rich.console import Console + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.shell.run import run_cmd + +console = Console() + + +def _image_tag(ctx: ScaffoldContext, svc_name: str) -> str: + """Full Artifact Registry image tag for a service.""" + region = ctx.region + project_id = ctx.project_id + repo = f"{ctx.project_name.lower().replace(' ', '-')}-docker" + return f"{region}-docker.pkg.dev/{project_id}/{repo}/{svc_name}:latest" + + +@register_step +class DockerBuildPush(BaseStep): + step_id = "L.1_docker_build_push" + phase = "L" + depends_on = ["K.3_api_docker_build", "G.3_create_artifact_registry"] + max_retries = 1 + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + region = ctx.region + + # Authenticate Docker with Artifact Registry via access token. + # This avoids requiring docker-credential-gcloud in PATH. + token = run_cmd("gcloud auth print-access-token", timeout=15) + if not token.ok: + raise RuntimeError("Failed to get gcloud access token for Docker auth") + login_result = run_cmd( + f"docker login -u oauth2accesstoken -p {token.stdout.strip()} " + f"{region}-docker.pkg.dev", + timeout=30, + ) + if not login_result.ok: + raise RuntimeError(f"Docker login failed: {login_result.stderr}") + + pushed = [] + svc_map = { + "api": "services/api", + "webapp": "services/web", + "worker": "services/worker", + } + + for svc_type, svc_dir_rel in svc_map.items(): + svc = ctx.service(svc_type) + if not svc: + continue + + svc_name = svc.name + svc_dir = str(ctx.project_dir / svc_dir_rel) + tag = _image_tag(ctx, svc_name) + + console.print(f" [cyan]building[/cyan] {svc_name} → {tag}") + run_cmd( + f"docker build --platform linux/amd64 -t {tag} .", + cwd=svc_dir, + check=True, + timeout=600, + ) + + console.print(f" [cyan]pushing[/cyan] {svc_name}") + run_cmd(f"docker push {tag}", check=True, timeout=300) + pushed.append(svc_name) + + return {"pushed": pushed} + + +@register_step +class TerraformApply(BaseStep): + step_id = "L.2_terraform_apply" + phase = "L" + depends_on = ["L.1_docker_build_push", "K.5_terraform_validate"] + max_retries = 0 + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + tf_dir = str(ctx.project_dir / "infra" / "terraform" / "envs" / "dev") + project_id = ctx.project_id + + # Ensure init has been run + run_cmd("terraform init", cwd=tf_dir, check=True, timeout=180) + + # Import resources that were already created via gcloud in earlier + # phases so Terraform doesn't try to re-create them and fail. + # Only import secrets that actually exist (skipped if DB phase was disabled). + imports = [] + for secret_name, tf_addr in [ + ("mongodb-uri", "module.secrets.google_secret_manager_secret.mongodb_uri"), + ("db-name", "module.secrets.google_secret_manager_secret.db_name"), + ]: + exists = run_cmd( + f"gcloud secrets describe {secret_name} --project={project_id}", + timeout=15, + ) + if exists.ok: + imports.append(( + secret_name, tf_addr, + f"projects/{project_id}/secrets/{secret_name}", + )) + + # Service accounts + prefix = ctx.project_name.lower().replace(" ", "-") + for role in ("build", "runtime"): + sa_name = f"{prefix}-{role}" + imports.append(( + sa_name, + f"module.iam.google_service_account.{role}", + f"projects/{project_id}/serviceAccounts/{sa_name}@{project_id}.iam.gserviceaccount.com", + )) + + for label, tf_addr, resource_id in imports: + check = run_cmd(f"terraform state show {tf_addr}", cwd=tf_dir, timeout=30) + if not check.ok: + console.print(f" [dim]importing[/dim] {label}") + run_cmd( + f"terraform import {tf_addr} {resource_id}", + cwd=tf_dir, + timeout=60, + ) + + result = run_cmd( + "terraform apply -auto-approve -input=false", + cwd=tf_dir, + check=True, + timeout=600, + ) + + # Capture outputs (service URLs) + out = run_cmd("terraform output -json", cwd=tf_dir, timeout=30) + outputs = {} + if out.ok: + try: + raw = json.loads(out.stdout) + outputs = {k: v.get("value", "") for k, v in raw.items()} + except (json.JSONDecodeError, AttributeError): + pass + + ctx.set("terraform_outputs", outputs) + + # Grab deployed URLs/job info from gcloud for verification. + # Workers are Cloud Run Jobs (no URL) — we report the job + scheduler. + deployed = {} + for svc_type, svc_name in [("api", "api"), ("webapp", "app")]: + svc = ctx.service(svc_type) + if not svc: + continue + url_result = run_cmd( + f"gcloud run services describe {svc_name} " + f"--region={ctx.region} --project={project_id} " + f"--format='value(status.url)'", + timeout=30, + ) + if url_result.ok and url_result.stdout.strip(): + url = url_result.stdout.strip().strip("'") + deployed[svc_name] = url + console.print(f" [green]live[/green] {svc_name} → {url}") + + # Check worker job exists + worker_svc = ctx.service("worker") + if worker_svc: + job_result = run_cmd( + f"gcloud run jobs describe {worker_svc.name} " + f"--region={ctx.region} --project={project_id} " + f"--format='value(name)'", + timeout=30, + ) + if job_result.ok and job_result.stdout.strip(): + console.print( + f" [green]live[/green] {worker_svc.name} (Cloud Run Job, schedule: {worker_svc.schedule})" + ) + deployed[worker_svc.name] = f"job:{worker_svc.name}" + + ctx.set("deployed_urls", deployed) + return {"apply_ok": True, "outputs": outputs, "deployed": deployed} + + +@register_step +class VerifyDeployment(BaseStep): + step_id = "L.3_verify_deployment" + phase = "L" + depends_on = ["L.2_terraform_apply"] + max_retries = 2 + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + deployed = ctx.get("deployed_urls", {}) + results = {} + + for svc_name, url in deployed.items(): + if not url: + results[svc_name] = "no_url" + continue + + # Workers are Cloud Run Jobs — skip HTTP health check + if url.startswith("job:"): + results[svc_name] = "ok (job)" + console.print(f" {svc_name}: [green]ok[/green] (Cloud Run Job — no URL to check)") + continue + + # Health check — try /health for api, / for others + check_path = "/health" if svc_name == "api" else "/" + check_url = f"{url}{check_path}" + + ok = False + last_code = "" + for attempt in range(6): + r = run_cmd( + f"curl -sf -o /dev/null -w '%{{http_code}}' {check_url}", + timeout=15, + ) + last_code = r.stdout.strip().strip("'") + if last_code.startswith("2"): + ok = True + break + time.sleep(5) + + results[svc_name] = "ok" if ok else f"failed (last status: {last_code})" + status = "[green]ok[/green]" if ok else f"[red]failed ({last_code})[/red]" + console.print(f" {svc_name}: {status} {check_url}") + + failures = [k for k, v in results.items() if v != "ok"] + if failures: + console.print( + f"[yellow]Warning: {', '.join(failures)} did not return 2xx. " + f"Services are deployed but may need debugging.[/yellow]" + ) + return {"verification": results} + + +@register_step +class WriteReadmeWithEndpoints(BaseStep): + """Regenerate README.md with live endpoint URLs after deployment.""" + + step_id = "L.4_write_readme" + phase = "L" + depends_on = ["L.3_verify_deployment"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + + # Get project number for Cloud Run URLs + r = run_cmd( + f"gcloud projects describe {project_id} --format='value(projectNumber)'" + ) + project_number = r.stdout.strip().strip("'") if r.ok else "PROJECTNUM" + + from platform_cli.templates.renderer import render_template + content = render_template( + "project/README.md.j2", + manifest=ctx.manifest, + ) + # Replace placeholder with real project number + content = content.replace("PROJECTNUM", project_number) + + readme_path = ctx.project_dir / "README.md" + readme_path.write_text(content) + console.print(" [green]wrote[/green] README.md with live endpoints") + return {"readme_updated": True} diff --git a/repoScaffold/src/platform_cli/steps/phase_m_pipelines.py b/repoScaffold/src/platform_cli/steps/phase_m_pipelines.py new file mode 100644 index 0000000..9690461 --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_m_pipelines.py @@ -0,0 +1,522 @@ +"""Phase M: CI/CD pipeline setup — GitHub repo, Cloud Build connection, triggers. + + M.1 grant_cloudbuild_permissions — grant Cloud Build SA the required roles + M.2 create_github_repo — create GitHub repo and push code + M.3 connect_github_to_cloudbuild — create Cloud Build GitHub connection + link repo + M.4 create_build_triggers — create per-service push triggers in Cloud Build + M.5 run_initial_builds — submit first builds via Cloud Build +""" +from __future__ import annotations + +import json +import time +from typing import Any + +from rich.console import Console + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.shell.run import run_cmd + +console = Console() + + +@register_step +class GrantCloudBuildPermissions(BaseStep): + step_id = "M.1_grant_cloudbuild_permissions" + phase = "M" + depends_on = ["G.2_enable_apis"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + + # Get the Cloud Build service account (project-number@cloudbuild) + r = run_cmd( + f"gcloud projects describe {project_id} --format='value(projectNumber)'" + ) + if not r.ok: + raise RuntimeError(f"Could not get project number: {r.stderr}") + project_number = r.stdout.strip().strip("'") + cb_sa = f"{project_number}@cloudbuild.gserviceaccount.com" + + roles = [ + "roles/run.admin", + "roles/iam.serviceAccountUser", + "roles/secretmanager.secretAccessor", + "roles/artifactregistry.writer", + "roles/compute.admin", + "roles/vpcaccess.admin", + "roles/iam.securityAdmin", + ] + for role in roles: + run_cmd( + f"gcloud projects add-iam-policy-binding {project_id} " + f"--member=serviceAccount:{cb_sa} " + f"--role={role} --quiet" + ) + + # The Cloud Build P4SA (per-project service agent) needs Secret + # Manager permissions to store GitHub OAuth tokens for connections. + p4sa = f"service-{project_number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" + p4sa_roles = [ + "roles/secretmanager.admin", + ] + for role in p4sa_roles: + run_cmd( + f"gcloud projects add-iam-policy-binding {project_id} " + f"--member=serviceAccount:{p4sa} " + f"--role={role} --quiet" + ) + + ctx.set("cloudbuild_sa", cb_sa) + return {"cloudbuild_sa": cb_sa, "roles_granted": roles} + + +@register_step +class CreateGithubRepo(BaseStep): + """Create a GitHub repo and push the project code.""" + + step_id = "M.2_create_github_repo" + phase = "M" + depends_on = ["M.1_grant_cloudbuild_permissions"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_dir = str(ctx.project_dir) + repo_name = ctx.project_name.lower().replace(" ", "-") + + # Check gh auth + auth = run_cmd("gh auth status") + if not auth.ok: + raise RuntimeError( + "GitHub CLI is not authenticated. Run `gh auth login` and re-run." + ) + + # Extract GitHub username/org + who = run_cmd("gh api user --jq '.login'") + gh_owner = who.stdout.strip() if who.ok else "" + + # Initialize git if needed + check = run_cmd("git rev-parse --git-dir", cwd=project_dir) + if not check.ok: + run_cmd("git init", cwd=project_dir, check=True) + + # Stage and commit if there are uncommitted changes + status = run_cmd("git status --porcelain", cwd=project_dir) + if status.stdout.strip(): + run_cmd("git add -A", cwd=project_dir) + run_cmd( + 'git commit -m "Initial scaffold via platform-cli"', + cwd=project_dir, + ) + + # Create GitHub repo if it doesn't exist + repo_check = run_cmd( + f"gh repo view {gh_owner}/{repo_name} --json name", + ) + if repo_check.ok: + console.print(f" [dim]exists[/dim] {gh_owner}/{repo_name}") + else: + run_cmd( + f"gh repo create {repo_name} --private --source={project_dir} --push", + cwd=project_dir, + check=True, + timeout=60, + ) + console.print(f" [green]created[/green] {gh_owner}/{repo_name}") + + # Ensure remote is set and push + remote_check = run_cmd("git remote get-url origin", cwd=project_dir) + if not remote_check.ok: + run_cmd( + f"git remote add origin https://github.com/{gh_owner}/{repo_name}.git", + cwd=project_dir, + ) + run_cmd("git push -u origin HEAD", cwd=project_dir, timeout=120) + + ctx.set("github_owner", gh_owner) + ctx.set("github_repo", repo_name) + return {"owner": gh_owner, "repo": repo_name} + + +@register_step +class ConnectGithubToCloudBuild(BaseStep): + """Create a Cloud Build 2nd-gen connection to GitHub and link the repo.""" + + step_id = "M.3_connect_github_to_cloudbuild" + phase = "M" + depends_on = ["M.2_create_github_repo"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + region = ctx.region + gh_owner = ctx.get("github_owner", "") + gh_repo = ctx.get("github_repo", "") + connection_name = f"{ctx.project_name.lower().replace(' ', '-')}-github" + + # Check if connection already exists + check = run_cmd( + f"gcloud builds connections describe {connection_name} " + f"--project={project_id} --region={region}", + ) + + if not check.ok: + # Create the connection — this starts an OAuth flow + console.print( + f" [cyan]creating[/cyan] Cloud Build GitHub connection: {connection_name}" + ) + console.print( + " [yellow]A browser window will open for GitHub authorization.[/yellow]" + ) + console.print( + " [yellow]1. Authorize Cloud Build to access your GitHub account[/yellow]" + ) + console.print( + " [yellow]2. Install the Cloud Build app on the repository[/yellow]" + ) + + result = run_cmd( + f"gcloud builds connections create github {connection_name} " + f"--project={project_id} --region={region}", + timeout=300, + ) + if not result.ok: + raise RuntimeError( + f"Failed to create GitHub connection: {result.stderr}" + ) + + # Poll until the connection installation is COMPLETE + console.print( + " [cyan]waiting[/cyan] for GitHub app installation to complete..." + ) + deadline = time.time() + 300 # 5 min timeout + while time.time() < deadline: + desc = run_cmd( + f"gcloud builds connections describe {connection_name} " + f"--project={project_id} --region={region} " + f"--format='value(installationState.stage)'", + ) + stage = desc.stdout.strip().strip("'") + if stage == "COMPLETE": + console.print(" [green]connected[/green] GitHub ↔ Cloud Build") + break + if stage == "PENDING_USER_OAUTH": + console.print( + " [yellow]waiting for OAuth authorization in browser...[/yellow]" + ) + elif stage == "PENDING_INSTALL_APP": + console.print( + " [yellow]waiting for Cloud Build app installation on GitHub...[/yellow]" + ) + time.sleep(10) + else: + console.print( + "[yellow]Warning: Connection not yet complete. " + "Finish setup in Cloud Console → Cloud Build → Repositories.[/yellow]" + ) + return {"connection": connection_name, "status": "incomplete"} + else: + console.print(f" [dim]exists[/dim] connection: {connection_name}") + + # Link the specific GitHub repo to the connection + linked_repo_name = f"{gh_owner}-{gh_repo}" + repo_check = run_cmd( + f"gcloud builds repositories describe {linked_repo_name} " + f"--connection={connection_name} " + f"--project={project_id} --region={region}", + ) + if not repo_check.ok: + run_cmd( + f"gcloud builds repositories create {linked_repo_name} " + f"--connection={connection_name} " + f"--remote-uri=https://github.com/{gh_owner}/{gh_repo}.git " + f"--project={project_id} --region={region}", + check=True, + timeout=60, + ) + console.print(f" [green]linked[/green] repo: {gh_owner}/{gh_repo}") + else: + console.print(f" [dim]exists[/dim] linked repo: {linked_repo_name}") + + ctx.set("cloudbuild_connection", connection_name) + ctx.set("cloudbuild_repo", linked_repo_name) + return { + "connection": connection_name, + "linked_repo": linked_repo_name, + "status": "complete", + } + + +@register_step +class CreateBuildTriggers(BaseStep): + """Create Cloud Build push triggers in GCP connected to the GitHub repo.""" + + step_id = "M.4_create_build_triggers" + phase = "M" + depends_on = ["M.3_connect_github_to_cloudbuild", "I.1_write_api_build"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + region = ctx.region + connection = ctx.get("cloudbuild_connection", "") + linked_repo = ctx.get("cloudbuild_repo", "") + + if not connection or not linked_repo: + console.print( + " [yellow]skipping[/yellow] trigger creation — no repo connection" + ) + return {"triggers": [], "note": "no connection"} + + repo_path = ( + f"projects/{project_id}/locations/{region}/" + f"connections/{connection}/repositories/{linked_repo}" + ) + + # Use the project's build SA (created in Phase G) + prefix = ctx.project_name.lower().replace(" ", "-") + cb_sa = f"projects/{project_id}/serviceAccounts/{prefix}-build@{project_id}.iam.gserviceaccount.com" + + triggers_created = [] + + # Service triggers + Terraform infra trigger + svc_configs = [] + for svc_type, config, files in [ + ("api", "cloudbuild/api.yaml", ["services/api/**", "cloudbuild/api.yaml"]), + ("webapp", "cloudbuild/web.yaml", ["services/web/**", "cloudbuild/web.yaml"]), + ("worker", "cloudbuild/worker.yaml", ["services/worker/**", "cloudbuild/worker.yaml"]), + ]: + svc = ctx.service(svc_type) + if svc: + svc_configs.append((f"build-{svc.name}", config, files)) + + # Always add the Terraform infra trigger + svc_configs.append(( + "deploy-infra", + "cloudbuild/terraform.yaml", + ["infra/**", "cloudbuild/terraform.yaml"], + )) + + for trigger_name, config_path, included_files in svc_configs: + + # Check if trigger already exists + check = run_cmd( + f"gcloud builds triggers describe {trigger_name} " + f"--project={project_id} --region={region}", + ) + if check.ok: + console.print(f" [dim]exists[/dim] {trigger_name}") + triggers_created.append(trigger_name) + continue + + # 2nd-gen repo triggers require serviceAccount — use REST API + # because gcloud CLI doesn't support serviceAccount flag with + # the `create github` subcommand. + token = run_cmd("gcloud auth print-access-token", timeout=15) + if not token.ok: + raise RuntimeError("Failed to get access token for trigger creation") + + included_json = json.dumps(included_files) + payload = json.dumps({ + "name": trigger_name, + "serviceAccount": cb_sa, + "repositoryEventConfig": { + "repository": repo_path, + "push": {"branch": "^main$"}, + }, + "filename": config_path, + "includedFiles": included_files, + }) + + result = run_cmd( + f"curl -sf -X POST " + f"'https://cloudbuild.googleapis.com/v1/projects/{project_id}" + f"/locations/{region}/triggers' " + f"-H 'Authorization: Bearer {token.stdout.strip()}' " + f"-H 'Content-Type: application/json' " + f"-d '{payload}'", + timeout=30, + ) + + if result.ok and "error" not in result.stdout.lower(): + console.print(f" [green]created[/green] trigger: {trigger_name}") + triggers_created.append(trigger_name) + else: + console.print( + f" [yellow]warning[/yellow] could not create trigger " + f"{trigger_name}: {(result.stderr or result.stdout)[:300]}" + ) + + return {"triggers": triggers_created} + + +@register_step +class RunInitialBuilds(BaseStep): + """Submit the Cloud Build configs to build and deploy each service.""" + + step_id = "M.5_run_initial_builds" + phase = "M" + depends_on = ["M.4_create_build_triggers"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + region = ctx.region + project_dir = str(ctx.project_dir) + + builds_submitted = [] + svc_configs = [ + ("api", "cloudbuild/api.yaml"), + ("webapp", "cloudbuild/web.yaml"), + ("worker", "cloudbuild/worker.yaml"), + ] + + for svc_type, config_path in svc_configs: + svc = ctx.service(svc_type) + if not svc: + continue + + console.print(f" [cyan]submitting[/cyan] build for {svc.name}") + + result = run_cmd( + f"gcloud builds submit " + f"--config={config_path} " + f"--project={project_id} " + f"--region={region} " + f"--substitutions=SHORT_SHA=initial " + f"--async " + f".", + cwd=project_dir, + timeout=120, + ) + + if result.ok: + builds_submitted.append(svc.name) + console.print(f" [green]submitted[/green] {svc.name}") + else: + console.print( + f" [yellow]warning[/yellow] build submit failed for " + f"{svc.name}: {result.stderr[:200]}" + ) + + return {"builds": builds_submitted} + + +@register_step +class CreateAgentDeployKey(BaseStep): + """Generate an SSH deploy key for agent access to the GitHub repo. + + Creates an SSH key pair, adds the public key as a deploy key on the + GitHub repo (with write access), and stores the private key in GCP + Secret Manager so the agent VM can retrieve it. + """ + + step_id = "M.6_create_agent_deploy_key" + phase = "M" + depends_on = ["M.2_create_github_repo", "N.1_grant_agent_access"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + import tempfile + + project_id = ctx.project_id + project_dir = str(ctx.project_dir) + key_name = f"{ctx.project_name.lower().replace(' ', '-')}-agent-deploy-key" + secret_name = "github-deploy-key" + + # Derive owner/repo from git remote (resilient to resume) + gh_owner = ctx.get("github_owner", "") + gh_repo = ctx.get("github_repo", "") + if not gh_owner or not gh_repo: + remote = run_cmd("git remote get-url origin", cwd=project_dir) + if remote.ok: + # Parse https://github.com/owner/repo.git or git@github.com:owner/repo.git + url = remote.stdout.strip() + if "github.com" in url: + parts = url.rstrip(".git").split("github.com")[-1] + parts = parts.lstrip("/:").split("/") + if len(parts) >= 2: + gh_owner, gh_repo = parts[0], parts[1] + + if not gh_owner or not gh_repo: + console.print(" [yellow]skipping[/yellow] — no GitHub repo") + return {"deploy_key": None} + + # Check if the secret already exists (key already generated) + check = run_cmd( + f"gcloud secrets describe {secret_name} --project={project_id}" + ) + if check.ok: + console.print(f" [dim]exists[/dim] deploy key in Secret Manager") + + # Verify the deploy key is still on the repo + keys = run_cmd( + f"gh api repos/{gh_owner}/{gh_repo}/keys --jq '.[].title'" + ) + if keys.ok and key_name in keys.stdout: + console.print(f" [dim]exists[/dim] deploy key on GitHub repo") + return {"deploy_key": secret_name, "status": "exists"} + + # Generate SSH key pair + with tempfile.TemporaryDirectory() as tmpdir: + key_path = f"{tmpdir}/deploy_key" + run_cmd( + f"ssh-keygen -t ed25519 -C '{key_name}' -f {key_path} -N ''", + check=True, + timeout=15, + ) + + # Read the keys + with open(key_path) as f: + private_key = f.read() + with open(f"{key_path}.pub") as f: + public_key = f.read().strip() + + # Add public key as deploy key on GitHub repo (with write access) + result = run_cmd( + f"gh api repos/{gh_owner}/{gh_repo}/keys " + f"-f title='{key_name}' " + f"-f key='{public_key}' " + f"-f read_only=false", + timeout=30, + ) + if not result.ok: + console.print( + f" [yellow]warning[/yellow] could not add deploy key: " + f"{result.stderr[:200]}" + ) + return {"deploy_key": None, "error": result.stderr[:200]} + + console.print(f" [green]added[/green] deploy key to {gh_owner}/{gh_repo}") + + # Store private key in GCP Secret Manager + if not check.ok: + run_cmd( + f"gcloud secrets create {secret_name} " + f"--project={project_id} --replication-policy=automatic", + timeout=30, + ) + + # Write private key to a temp file and pipe to gcloud + import tempfile as tf + with tf.NamedTemporaryFile(mode="w", suffix=".pem", delete=False) as tmp: + tmp.write(private_key) + tmp_path = tmp.name + + run_cmd( + f"gcloud secrets versions add {secret_name} " + f"--data-file={tmp_path} --project={project_id}", + check=True, + timeout=30, + ) + + import os + os.unlink(tmp_path) + + console.print( + f" [green]stored[/green] private key in Secret Manager: {secret_name}" + ) + console.print( + f" [dim]agent retrieves it via:[/dim] " + f"gcloud secrets versions access latest " + f"--secret={secret_name} --project={project_id}" + ) + + return {"deploy_key": secret_name, "status": "created"} diff --git a/repoScaffold/src/platform_cli/steps/phase_n_agents.py b/repoScaffold/src/platform_cli/steps/phase_n_agents.py new file mode 100644 index 0000000..5ddc31b --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_n_agents.py @@ -0,0 +1,156 @@ +"""Phase N: Agent access — grant service accounts access to project resources. + + N.1 grant_agent_access — grant each agent SA access to Secret Manager, + Cloud Run, Artifact Registry, and Cloud Build so agents can test + and deploy without direct GCP console access. + N.2 store_claude_oauth_token — persist the Claude Code subscription OAuth + token in Secret Manager so the agent VM can authenticate Claude Code via + a long-lived token (from `claude setup-token`, valid ~1 year, drawing on + the Claude subscription rather than metered API billing) instead of an + expiring interactive login. +""" +from __future__ import annotations + +import os +import tempfile +from typing import Any + +from rich.console import Console + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.manifest.defaults import DEFAULT_AGENT_SERVICE_ACCOUNT +from platform_cli.shell.run import run_cmd + +console = Console() + +# Secret Manager name for the Claude Code subscription OAuth token used by agents. +CLAUDE_OAUTH_SECRET_NAME = "claude-code-oauth-token" + + +def _get_claude_oauth_token() -> str: + """Retrieve the Claude Code subscription OAuth token from the environment. + + Sourced from CLAUDE_CODE_OAUTH_TOKEN (produced by `claude setup-token`) so a + token is never hard-coded or committed. Returns an empty string if unset + (the step then skips with guidance). + """ + return os.environ.get("CLAUDE_CODE_OAUTH_TOKEN", "").strip() + + +@register_step +class GrantAgentAccess(BaseStep): + step_id = "N.1_grant_agent_access" + phase = "N" + depends_on = ["G.2_enable_apis", "H.1_write_secret_manifest"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + + # Merge manifest SAs with the default agent SA + agent_sas = list(ctx.manifest.agents.service_accounts) + if DEFAULT_AGENT_SERVICE_ACCOUNT and DEFAULT_AGENT_SERVICE_ACCOUNT not in agent_sas: + agent_sas.append(DEFAULT_AGENT_SERVICE_ACCOUNT) + + if not agent_sas: + console.print(" [dim]no agent service accounts configured[/dim]") + return {"agents": []} + + # Roles that give agents the ability to: + # - Read secrets (for testing with real config) — NO write access + # - View and invoke Cloud Run services + # - Submit Cloud Builds + # - Read Artifact Registry images + # - View project resources + roles = [ + "roles/secretmanager.secretAccessor", + "roles/run.invoker", + "roles/run.viewer", + "roles/cloudbuild.builds.editor", + "roles/artifactregistry.reader", + "roles/viewer", + ] + + granted = [] + for sa in agent_sas: + console.print(f" [cyan]granting[/cyan] access to {sa}") + + for role in roles: + run_cmd( + f"gcloud projects add-iam-policy-binding {project_id} " + f"--member=serviceAccount:{sa} " + f"--role={role} --quiet" + ) + + granted.append(sa) + console.print(f" [green]granted[/green] {len(roles)} roles to {sa}") + + return {"agents": granted, "roles": roles} + + +@register_step +class StoreClaudeOAuthToken(BaseStep): + """Persist the Claude Code subscription OAuth token in Secret Manager. + + Agents run Claude Code headlessly on the VM. Interactive logins expire + (taking the whole fleet down when they lapse), so we store a long-lived + subscription OAuth token (from ``claude setup-token``, valid ~1 year and + billed to the Claude subscription rather than metered API usage) in Secret + Manager and let the VM read it on every iteration. Agent SAs already receive + project-level ``secretmanager.secretAccessor`` in N.1, so no per-secret + grant is needed. + """ + + step_id = "N.2_store_claude_oauth_token" + phase = "N" + depends_on = ["G.2_enable_apis"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + token = _get_claude_oauth_token() + + if not token: + console.print( + " [yellow]skipping[/yellow] — no CLAUDE_CODE_OAUTH_TOKEN in the " + "environment.\n" + f" Create the secret later so agents can authenticate:\n" + f" [dim]printf %s \"$(claude setup-token)\" | gcloud secrets " + f"create {CLAUDE_OAUTH_SECRET_NAME} --data-file=- " + f"--replication-policy=automatic --project={project_id}[/dim]" + ) + return {"stored": False, "secret": CLAUDE_OAUTH_SECRET_NAME} + + exists = run_cmd( + f"gcloud secrets describe {CLAUDE_OAUTH_SECRET_NAME} " + f"--project={project_id}" + ).ok + if not exists: + run_cmd( + f"gcloud secrets create {CLAUDE_OAUTH_SECRET_NAME} " + f"--project={project_id} --replication-policy=automatic", + check=True, + timeout=30, + ) + + # Write the token to a temp file to avoid exposing it in process args. + with tempfile.NamedTemporaryFile( + "w", delete=False, prefix="claude-oauth-", suffix=".token" + ) as tmp: + tmp.write(token) + tmp_path = tmp.name + try: + run_cmd( + f"gcloud secrets versions add {CLAUDE_OAUTH_SECRET_NAME} " + f"--data-file={tmp_path} --project={project_id}", + check=True, + timeout=30, + ) + finally: + os.unlink(tmp_path) + + console.print( + f" [green]stored[/green] {CLAUDE_OAUTH_SECRET_NAME} " + "(agents authenticate Claude via subscription OAuth token)" + ) + return {"stored": True, "secret": CLAUDE_OAUTH_SECRET_NAME} diff --git a/repoScaffold/src/platform_cli/steps/phase_o_linear.py b/repoScaffold/src/platform_cli/steps/phase_o_linear.py new file mode 100644 index 0000000..d6f97b2 --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_o_linear.py @@ -0,0 +1,342 @@ +"""Phase O: Linear project management integration. + + O.1 ensure_linear_team — find or create the Linear team + O.2 setup_github_integration — link Linear ↔ GitHub for auto PR/branch linking + O.3 create_initial_issues — seed the board with initial project tasks +""" +from __future__ import annotations + +import json +import os +from typing import Any + +from rich.console import Console + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.shell.run import run_cmd + +console = Console() + + +def _get_linear_api_key() -> str: + """Retrieve the Linear API key from ~/.linear/ config or env var.""" + key = os.environ.get("LINEAR_API_KEY", "") + if key: + return key + + config_path = os.path.expanduser("~/.linear/config.json") + if os.path.exists(config_path): + try: + with open(config_path) as f: + config = json.load(f) + return config.get("apiKey", "") + except (json.JSONDecodeError, KeyError): + pass + return "" + + +def _linear_gql(query: str, variables: dict | None = None) -> dict: + """Execute a Linear GraphQL query/mutation.""" + api_key = _get_linear_api_key() + if not api_key: + raise RuntimeError( + "Linear not authenticated. Run `lin` to enter your API key, " + "or set LINEAR_API_KEY env var." + ) + + payload = json.dumps({"query": query, "variables": variables or {}}) + # Escape single quotes in payload for shell + payload_escaped = payload.replace("'", "'\\''") + + r = run_cmd( + f"curl -sf -X POST https://api.linear.app/graphql " + f"-H 'Authorization: {api_key}' " + f"-H 'Content-Type: application/json' " + f"-d '{payload_escaped}'", + timeout=30, + ) + if not r.ok: + raise RuntimeError(f"Linear API error: {r.stderr}") + + data = json.loads(r.stdout) + if "errors" in data: + raise RuntimeError(f"Linear GraphQL error: {data['errors']}") + return data.get("data", {}) + + +@register_step +class EnsureLinearTeam(BaseStep): + """Find or create the Linear team specified in the manifest.""" + + step_id = "O.1_ensure_linear_team" + phase = "O" + depends_on = ["A.3_authenticate"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not ctx.manifest.linear.enabled + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + linear = ctx.manifest.linear + team_key = linear.team_key + team_name = linear.team_name or ctx.project_name + + # Search for existing team by key + data = _linear_gql(""" + query($key: String!) { + teams(filter: { key: { eq: $key } }) { + nodes { id name key } + } + } + """, {"key": team_key}) + + teams = data.get("teams", {}).get("nodes", []) + if teams: + team = teams[0] + console.print(f" [dim]exists[/dim] team: {team['key']} ({team['name']})") + ctx.set("linear_team_id", team["id"]) + return {"team_id": team["id"], "team_key": team["key"]} + + # Create team + data = _linear_gql(""" + mutation($name: String!, $key: String!) { + teamCreate(input: { name: $name, key: $key }) { + success + team { id name key } + } + } + """, {"name": team_name, "key": team_key}) + + result = data.get("teamCreate", {}) + if not result.get("success"): + raise RuntimeError(f"Failed to create Linear team: {data}") + + team = result["team"] + console.print(f" [green]created[/green] team: {team['key']} ({team['name']})") + ctx.set("linear_team_id", team["id"]) + return {"team_id": team["id"], "team_key": team["key"]} + + +@register_step +class SetupGithubIntegration(BaseStep): + """Verify that Linear ↔ GitHub integration is active. + + Linear's GitHub integration must be installed via the Linear UI + (Settings → Integrations → GitHub). This step checks if it's + connected and provides instructions if not. + """ + + step_id = "O.2_setup_github_integration" + phase = "O" + depends_on = ["O.1_ensure_linear_team", "M.2_create_github_repo"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not ctx.manifest.linear.enabled + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + # Check for active integrations + data = _linear_gql(""" + query { + integrations { + nodes { id service } + } + } + """) + + integrations = data.get("integrations", {}).get("nodes", []) + github_connected = any( + i.get("service") == "github" for i in integrations + ) + + if github_connected: + console.print(" [dim]exists[/dim] Linear ↔ GitHub integration") + else: + workspace = ctx.manifest.linear.workspace + console.print( + f" [yellow]action needed[/yellow] Connect GitHub to Linear:\n" + f" https://linear.app/{workspace}/settings/integrations/github\n" + f" This enables automatic PR/branch linking with issues." + ) + + return {"github_connected": github_connected} + + +@register_step +class CreateInitialIssues(BaseStep): + """Seed the Linear board with initial project setup tasks.""" + + step_id = "O.3_create_initial_issues" + phase = "O" + depends_on = ["O.2_setup_github_integration"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not ctx.manifest.linear.enabled + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + team_id = ctx.get("linear_team_id") + if not team_id: + return {"issues": [], "note": "no team"} + + project_name = ctx.project_name + gh_owner = ctx.get("github_owner", "") + gh_repo = ctx.get("github_repo", "") + repo_url = f"https://github.com/{gh_owner}/{gh_repo}" if gh_owner else "" + + # Check if issues already exist for this project + team_key = ctx.manifest.linear.team_key + data = _linear_gql(""" + query($teamId: String!) { + issues(filter: { team: { id: { eq: $teamId } } }, first: 5) { + nodes { id title } + } + } + """, {"teamId": team_id}) + + existing = data.get("issues", {}).get("nodes", []) + if len(existing) >= 3: + console.print( + f" [dim]exists[/dim] {len(existing)} issues already on board" + ) + return {"issues": [i["title"] for i in existing], "note": "pre-existing"} + + # Seed issues for the project + issues_to_create = [ + { + "title": f"Set up development environment for {project_name}", + "description": ( + f"## Objective\n" + f"Get the local development environment running for {project_name}.\n\n" + f"## Steps\n" + f"1. Clone the repo: `git clone {repo_url}`\n" + f"2. Run `docker compose up` to start all services\n" + f"3. Verify API at http://localhost:3006/health\n" + f"4. Verify web app at http://localhost:3005\n\n" + f"## Acceptance Criteria\n" + f"- All 3 services start without errors\n" + f"- API health check returns 200\n" + f"- Web app loads in browser" + ), + "priority": 2, + }, + { + "title": f"Review and document API endpoints for {project_name}", + "description": ( + f"## Objective\n" + f"Document all API endpoints with request/response schemas.\n\n" + f"## Steps\n" + f"1. Review `services/api/src/routes/` for all route definitions\n" + f"2. Create API documentation in `services/api/README.md`\n" + f"3. Include auth requirements, request params, response shapes\n\n" + f"## Branch\n" + f"Create branch: `docs/api-documentation`\n\n" + f"## Acceptance Criteria\n" + f"- Every endpoint documented with method, path, auth, params, response\n" + f"- README includes example curl commands" + ), + "priority": 3, + }, + { + "title": f"Set up monitoring and alerting for {project_name}", + "description": ( + f"## Objective\n" + f"Configure GCP monitoring for all Cloud Run services.\n\n" + f"## Steps\n" + f"1. Add uptime checks for API /health endpoint\n" + f"2. Configure alert policies for 5xx error rate > 1%\n" + f"3. Set up log-based metrics for key events\n" + f"4. Add Terraform resources in `infra/terraform/`\n\n" + f"## Branch\n" + f"Create branch: `feat/monitoring-alerting`\n\n" + f"## Acceptance Criteria\n" + f"- Uptime check pings /health every 60s\n" + f"- Alert fires on sustained 5xx errors\n" + f"- Changes deployed via Terraform pipeline" + ), + "priority": 3, + }, + ] + + created = [] + for issue in issues_to_create: + data = _linear_gql(""" + mutation($teamId: String!, $title: String!, $description: String, $priority: Int) { + issueCreate(input: { + teamId: $teamId, + title: $title, + description: $description, + priority: $priority + }) { + success + issue { id identifier title url } + } + } + """, { + "teamId": team_id, + "title": issue["title"], + "description": issue["description"], + "priority": issue["priority"], + }) + + result = data.get("issueCreate", {}) + if result.get("success"): + i = result["issue"] + created.append(i["identifier"]) + console.print( + f" [green]created[/green] {i['identifier']}: {i['title']}" + ) + + return {"issues": created} + + +@register_step +class StoreLinearKeyInSecretManager(BaseStep): + """Store the Linear API key in GCP Secret Manager for the agent VM.""" + + step_id = "O.4_store_linear_key" + phase = "O" + depends_on = ["O.1_ensure_linear_team"] + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not ctx.manifest.linear.enabled + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + secret_name = "linear-api-key" + api_key = _get_linear_api_key() + + if not api_key: + console.print( + " [yellow]skipping[/yellow] — no Linear API key available" + ) + return {"stored": False} + + # Create secret if it doesn't exist + check = run_cmd( + f"gcloud secrets describe {secret_name} --project={project_id}" + ) + if not check.ok: + run_cmd( + f"gcloud secrets create {secret_name} " + f"--project={project_id} --replication-policy=automatic", + timeout=30, + ) + + # Store the key + import tempfile + with tempfile.NamedTemporaryFile(mode="w", suffix=".key", delete=False) as f: + f.write(api_key) + tmp_path = f.name + + run_cmd( + f"gcloud secrets versions add {secret_name} " + f"--data-file={tmp_path} --project={project_id}", + check=True, + timeout=30, + ) + + os.unlink(tmp_path) + console.print( + f" [green]stored[/green] Linear API key in Secret Manager: {secret_name}" + ) + return {"stored": True, "secret": secret_name} diff --git a/repoScaffold/src/platform_cli/steps/phase_p_library_pipeline.py b/repoScaffold/src/platform_cli/steps/phase_p_library_pipeline.py new file mode 100644 index 0000000..325a10d --- /dev/null +++ b/repoScaffold/src/platform_cli/steps/phase_p_library_pipeline.py @@ -0,0 +1,197 @@ +"""Phase P: Library (npm) CI/CD pipeline. + +Library-only steps (self-skip for `app` projects). They create a slim GCP +project, an npm-token secret, and a Cloud Build trigger that runs the +`cloudbuild/publish.yaml` pipeline on push to main. + + P.1 lib_gcp_project — create/reuse the GCP project, link billing, enable + only cloudbuild + secretmanager APIs + P.2 lib_npm_secret — create the npm-token secret (value from NPM_TOKEN env + if present) and grant the build SAs access + P.3 lib_build_trigger — create the push-to-main Cloud Build trigger for the + existing GitHub repo (prints connect instructions if + the repo is not yet linked to Cloud Build) +""" +from __future__ import annotations + +import os +import tempfile +from typing import Any + +from rich.console import Console + +from platform_cli.engine.context import ScaffoldContext +from platform_cli.engine.registry import register_step +from platform_cli.engine.step import BaseStep +from platform_cli.shell.run import run_cmd +from platform_cli.steps._gcp_org import ensure_project_in_org, resolve_org_id + +console = Console() + + +class _LibraryStep(BaseStep): + """Base for library-only steps.""" + phase = "P" + + def should_skip(self, ctx: ScaffoldContext) -> bool: + return super().should_skip(ctx) or not ctx.manifest.project.is_library + + +@register_step +class LibGcpProject(_LibraryStep): + step_id = "P.1_lib_gcp_project" + depends_on = ["A.3_authenticate"] + max_retries = 0 + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + + who = run_cmd("gcloud config get-value account") + if not who.ok or not who.stdout.strip(): + raise RuntimeError("No gcloud account. Run `gcloud auth login` and retry.") + + # Which org should this project live under? (env → manifest → detect) + org_id = resolve_org_id(ctx) + + if run_cmd(f"gcloud projects describe {project_id}").ok: + console.print(f" [dim]exists[/dim] project {project_id}") + run_cmd(f"gcloud config set project {project_id}", check=True) + # Adopt an orphaned (no-org) project into the org. + ensure_project_in_org(project_id, org_id) + else: + create = f"gcloud projects create {project_id} --name={ctx.project_name}" + if org_id: + create += f" --organization={org_id}" + res = run_cmd(create) + if not res.ok: + raise RuntimeError( + f"Could not create project '{project_id}'. Set an existing " + f"project in the manifest or create it manually.\n{res.stderr}" + ) + console.print(f" [green]created[/green] project {project_id}") + run_cmd(f"gcloud config set project {project_id}", check=True) + ensure_project_in_org(project_id, org_id) + billing = run_cmd( + "gcloud billing accounts list --filter='OPEN=true' " + "--format='value(ACCOUNT_ID)' --limit=1" + ) + bid = billing.stdout.strip().split("\n")[0] if billing.ok else "" + if bid and run_cmd( + f"gcloud billing projects link {project_id} --billing-account={bid}" + ).ok: + console.print(f" [green]linked billing[/green] {bid}") + + # Libraries need only Cloud Build + Secret Manager (no run/registry/vpc). + for api in ("cloudbuild.googleapis.com", "secretmanager.googleapis.com"): + run_cmd(f"gcloud services enable {api} --project={project_id}", check=True) + console.print(" [green]enabled[/green] cloudbuild + secretmanager APIs") + return {"project_id": project_id} + + +@register_step +class LibNpmSecret(_LibraryStep): + step_id = "P.2_lib_npm_secret" + depends_on = ["P.1_lib_gcp_project"] + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + name = "npm-token" + + if not run_cmd(f"gcloud secrets describe {name} --project={project_id}").ok: + run_cmd( + f"gcloud secrets create {name} --project={project_id} " + "--replication-policy=automatic", + check=True, + ) + console.print(f" [green]created[/green] secret {name}") + else: + console.print(f" [dim]exists[/dim] secret {name}") + + token = os.environ.get("NPM_TOKEN", "").strip() + if token: + with tempfile.NamedTemporaryFile("w", delete=False, suffix=".tok") as f: + f.write(token) + tmp = f.name + try: + run_cmd( + f"gcloud secrets versions add {name} --data-file={tmp} " + f"--project={project_id}", + check=True, + ) + console.print(" [green]stored[/green] NPM_TOKEN value") + finally: + os.unlink(tmp) + else: + console.print( + " [yellow]note[/yellow] npm-token has no value yet — add one:\n" + f" [dim]printf %s \"$NPM_TOKEN\" | gcloud secrets versions add " + f"{name} --data-file=- --project={project_id}[/dim]" + ) + + # Grant the build service accounts read access to the token. + num = run_cmd( + f"gcloud projects describe {project_id} --format='value(projectNumber)'" + ).stdout.strip() + if num: + for sa in ( + f"{num}@cloudbuild.gserviceaccount.com", + f"{num}-compute@developer.gserviceaccount.com", + ): + run_cmd( + f"gcloud secrets add-iam-policy-binding {name} --project={project_id} " + f"--member=serviceAccount:{sa} " + "--role=roles/secretmanager.secretAccessor --quiet" + ) + return {"secret": name, "has_value": bool(token)} + + +@register_step +class LibBuildTrigger(_LibraryStep): + step_id = "P.3_lib_build_trigger" + depends_on = ["P.1_lib_gcp_project"] + max_retries = 0 + + def run(self, ctx: ScaffoldContext) -> dict[str, Any]: + project_id = ctx.project_id + repo = ctx.manifest.project.repo.strip() + if not repo or "/" not in repo: + console.print( + " [yellow]skip[/yellow] no project.repo set (owner/name) — " + "cannot wire the Cloud Build trigger." + ) + return {"trigger": None, "reason": "no repo"} + owner, name = repo.split("/", 1) + trigger_name = f"{name}-publish" + build_config = "cloudbuild/publish.yaml" + + # Already exists? + existing = run_cmd( + f"gcloud builds triggers describe {trigger_name} --project={project_id} " + "--region=global" + ) + if existing.ok: + console.print(f" [dim]exists[/dim] trigger {trigger_name}") + return {"trigger": trigger_name} + + res = run_cmd( + f"gcloud builds triggers create github " + f"--name={trigger_name} " + f"--repo-owner={owner} --repo-name={name} " + f'--branch-pattern="^main$" ' + f"--build-config={build_config} " + f"--project={project_id} --region=global" + ) + if res.ok: + console.print(f" [green]created[/green] trigger {trigger_name} (push to main)") + return {"trigger": trigger_name} + + # Most common failure: the repo is not connected to Cloud Build yet. + console.print( + f" [yellow]action needed[/yellow] connect {owner}/{name} to Cloud Build, " + "then re-run:\n" + f" https://console.cloud.google.com/cloud-build/triggers/connect?project={project_id}\n" + f" Install the Cloud Build GitHub App for {owner}/{name}, then re-run " + "scaffold (or create the trigger manually with the same flags).\n" + f" [dim]{res.stderr.strip()[:300]}[/dim]" + ) + return {"trigger": None, "reason": "repo not connected"} diff --git a/repoScaffold/src/platform_cli/templates/__init__.py b/repoScaffold/src/platform_cli/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/repoScaffold/src/platform_cli/templates/filters.py b/repoScaffold/src/platform_cli/templates/filters.py new file mode 100644 index 0000000..a0974a5 --- /dev/null +++ b/repoScaffold/src/platform_cli/templates/filters.py @@ -0,0 +1,20 @@ +"""Custom Jinja2 filters: snake_case, kebab_case, env_var.""" +from __future__ import annotations + +import re + + +def snake_case(value: str) -> str: + s = re.sub(r"[\s\-]+", "_", value) + s = re.sub(r"([A-Z])", r"_\1", s).lower() + return re.sub(r"_+", "_", s).strip("_") + + +def kebab_case(value: str) -> str: + s = re.sub(r"[\s_]+", "-", value) + s = re.sub(r"([A-Z])", r"-\1", s).lower() + return re.sub(r"-+", "-", s).strip("-") + + +def env_var(value: str) -> str: + return snake_case(value).upper() diff --git a/repoScaffold/src/platform_cli/templates/renderer.py b/repoScaffold/src/platform_cli/templates/renderer.py new file mode 100644 index 0000000..ec5943b --- /dev/null +++ b/repoScaffold/src/platform_cli/templates/renderer.py @@ -0,0 +1,33 @@ +"""Jinja2 environment + render_to_file helper.""" +from __future__ import annotations + +from pathlib import Path + +import jinja2 + +from platform_cli.templates.filters import snake_case, kebab_case, env_var + +# Templates live in repoScaffold/templates/ (sibling of src/) +_TEMPLATES_DIR = Path(__file__).resolve().parent.parent.parent.parent / "templates" + +_env = jinja2.Environment( + loader=jinja2.FileSystemLoader(str(_TEMPLATES_DIR)), + keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, +) +_env.filters["snake_case"] = snake_case +_env.filters["kebab_case"] = kebab_case +_env.filters["env_var"] = env_var + + +def render_template(template_name: str, **kwargs) -> str: + """Render a .j2 template to a string.""" + tmpl = _env.get_template(template_name) + return tmpl.render(**kwargs) + + +def render_to_file(template_name: str, dest: Path, **kwargs) -> None: + """Render a .j2 template and write it to *dest*.""" + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(render_template(template_name, **kwargs)) diff --git a/repoScaffold/templates/agents/developer/PROMPT.library-node.md.j2 b/repoScaffold/templates/agents/developer/PROMPT.library-node.md.j2 new file mode 100644 index 0000000..bbe12f9 --- /dev/null +++ b/repoScaffold/templates/agents/developer/PROMPT.library-node.md.j2 @@ -0,0 +1,107 @@ +You are an autonomous agent working on the **{{ manifest.project.name }}** project — a **publishable Node/npm package** (not a web app). + +## Step 1: Orient + +1. Read `agents/AGENTS.md` for the package layout, build/test/publish commands, and versioning rules. +2. `git checkout main && git pull` — always start clean on main. +3. `git stash drop` any stale stashes. Do NOT carry over work from previous iterations. +4. Remember: the package lives in `{{ manifest.library.package_dir }}/`. Always `cd {{ manifest.library.package_dir }}` before running package scripts. There is no server, database, or cloud runtime — do not try to deploy anything. + +## Step 2: Sync Linear issues with PR/merge status + +Reconcile Linear with GitHub before picking new work. + +```bash +gh pr list --state merged --limit 10 --json number,title,mergedAt +gh pr list --state closed --limit 10 --json number,title,closedAt +``` + +For each **merged PR**: extract the issue ID from the title (e.g. `{{ manifest.linear.team_key }}-21`). If the Linear issue is not already **Done**, move it to Done via the API. +For each **closed (not merged) PR**: if the Linear issue is In Review/In Progress, move it to **Canceled**. + +**Then check open PRs and their comments:** + +```bash +gh pr list --state open --json number,title,headRefName,comments +``` + +For each open PR: +0. If the linked Linear issue is in **Backlog**, leave the PR alone — the owner demoted it and is reviewing. +1. Read comments: `gh pr view --comments`. +2. If there are **unanswered human comments**: `gh pr checkout `, address the feedback, `cd {{ manifest.library.package_dir }} && {{ manifest.library.build_cmd }}` to verify, commit, push, and **reply to each comment** explaining what changed. +3. Otherwise leave it for the reviewer. + +Also check Linear issue comments on In Progress / In Review tasks and address them. Only move on to new tasks when all open PRs and in-progress issues have no unanswered feedback. + +```bash +LINEAR_KEY=$(cat ~/.linear/api_key) +TEAM_ID=$(curl -s -H "Authorization: $LINEAR_KEY" -H "Content-Type: application/json" \ + -d '{"query": "{ teams(filter: { key: { eq: \"{{ manifest.linear.team_key }}\" } }) { nodes { id } } }"}' \ + https://api.linear.app/graphql | python3 -c "import sys,json; print(json.load(sys.stdin)['data']['teams']['nodes'][0]['id'])") +``` + +## Step 3: Get your task from Linear + +**Board caps — check the `[board-status: ...]` line at the very top of this prompt first:** +- If `in_review` is **greater than** `max_in_review`, the review queue is full. Do **NOT** pick up a new Todo task or move anything to In Progress — more parallel work just creates merge conflicts. You should already have handled open-PR feedback in Step 2, so end the iteration. +- If `backlog` is **greater than** `max_backlog`, do **NOT** create any new Backlog tasks in the "no Todo tasks" branch below. +- (When *both* caps are exceeded the loop backs the agent off before Claude even runs, so if you're reading this at least one path is still open.) + +**Only pick tasks with state "Todo"** (type `unstarted`) — owner-approved. Never pick Backlog, Done, or Canceled. + +```bash +curl -s -H "Authorization: $LINEAR_KEY" -H "Content-Type: application/json" \ + -d "{\"query\": \"{ team(id: \\\"$TEAM_ID\\\") { issues(filter: { state: { type: { eq: \\\"unstarted\\\" } } }, first: 10, orderBy: priority) { nodes { id identifier title description priority state { name type } } } } }\"}" \ + https://api.linear.app/graphql +``` + +Before picking, check it doesn't already have an open PR (`gh pr list --state open`). If it does, iterate on that PR's branch instead of opening a second one. + +If there are **no "Todo" tasks**, do a deep scan of the package and create **up to 10 substantial tasks in Backlog** (search existing issues first to avoid dupes). For a library, good tasks are: +- Emit TypeScript `.d.ts` declarations so consumers get types. +- Add real unit/interaction tests for the public components. +- Add CI (build + test on PRs; publish on tag). +- New props / controlled variants / better defaults on the public API. +- Bundle health: tree-shaking, `sideEffects`, smaller output, fewer runtime deps. +- Storybook stories + README examples for every public export. + +Do NOT create trivial CSS/text tweaks. Add them to **Backlog** (not Todo) and end the iteration — the owner approves them. + +## Step 4: Do the work + +1. Move the issue to **In Progress** (state type `started`). +2. Branch: `{{ manifest.linear.team_key | lower }}-{number}/{short-desc}`. +3. If you need credentials you don't have, comment on the issue requesting them, move it back to **Todo**, and skip to the next task. Never hardcode secrets or commit an `.npmrc` token. +4. Make focused changes inside `{{ manifest.library.package_dir }}/`. Keep the public API stable unless the task is explicitly a breaking change. +5. **Verify locally:** + - `cd {{ manifest.library.package_dir }} && {{ manifest.library.build_cmd }}` **must succeed.** + - If a real test script exists, `{{ manifest.library.test_cmd }}` must pass. (If `test` is only a placeholder that errors, skip it — do not treat the placeholder failure as a blocker.) + - For UI/component changes, sanity-check in Storybook if the package has it. +6. **Commit** with the issue ID: `{{ manifest.linear.team_key }}-{number}: {short summary}`. + +## Step 5: Open a PR — do NOT publish and do NOT mark Done + +1. Push: `git push -u origin HEAD`. +2. `gh pr create --title "{{ manifest.linear.team_key }}-{number}: {title}" --body "Resolves {{ manifest.linear.team_key }}-{number}. Suggested version bump: patch|minor|major (with reason)."` +3. Move the Linear issue to **In Review**. + +**Never run `{{ manifest.library.publish_cmd }}`, never bump the published version on `main`, and never mark an issue Done unless its PR is merged.** Releases are the maintainer's call — propose the semver bump in the PR body and stop there. + +## Step 6: Keep working + +After opening a PR, return to Step 2 and repeat until: there are no more Todo tasks, remaining tasks are blocked on human input, or all open PRs need human review. Maximize useful work per iteration. + +## Rules + +1. **Linear is the source of truth.** Observe full state before acting. +2. **Never push directly to main.** Feature branch + PR only. +3. **Never mark issues Done unless their PR is merged** (verify via `gh pr list --state merged`). +4. **One PR per task.** Push new commits to the existing branch instead of opening a second PR. +5. **Only pick "Todo" tasks.** Never touch Backlog items — the owner controls Backlog → Todo. +6. **New tasks go to Backlog**, and search before creating to avoid dupes. +7. **Address PR/issue feedback before new tasks.** Reply to every human comment. +8. **Every PR must build.** Run `{{ manifest.library.build_cmd }}` from `{{ manifest.library.package_dir }}/` first. +9. **Do not publish, do not bump the published version, do not commit registry credentials.** +10. **Preserve the public API** unless the task explicitly asks for a breaking change; call it out as a major bump in the PR. +11. **If blocked on one task, move to the next** — don't stop the whole iteration. +12. **Respect the board caps.** Read the `[board-status]` line at the top. When `in_review > max_in_review`, don't start new tasks or move anything to In Progress; when `backlog > max_backlog`, don't create Backlog tasks. The review-queue cap counts Linear issues **In Review**, not open GitHub PRs — a PR the owner demoted to Backlog stays open on GitHub but is not review load. These caps prevent merge-conflict pileups and runaway cost — the loop backs you off entirely when both are exceeded. diff --git a/repoScaffold/templates/agents/developer/PROMPT.md.j2 b/repoScaffold/templates/agents/developer/PROMPT.md.j2 new file mode 100644 index 0000000..62045f8 --- /dev/null +++ b/repoScaffold/templates/agents/developer/PROMPT.md.j2 @@ -0,0 +1,337 @@ +You are an autonomous agent working on the **{{ manifest.project.name }}** project. + +## Step 1: Orient + +1. Read `agents/AGENTS.md` for project structure and conventions. +2. You run in your OWN private git worktree, already scrubbed to the latest `main` (detached HEAD) at the start of every iteration — no other agent shares it. Do **not** run `git checkout main` (main is checked out in the shared clone, so it will fail); you're already on a clean, up-to-date tree. Just create your task branch (Step 4) when you pick up work. +3. Do NOT carry over anything from a previous iteration. The worktree is already clean; only your task's files should ever appear in `git status`. + +## Step 2: Sync Linear issues with PR/merge status + +Before looking at new tasks, reconcile Linear with GitHub state. + +**Check all recently merged or closed PRs** and update their Linear issues: + +```bash +# Get recently merged PRs +gh pr list --state merged --limit 10 --json number,title,mergedAt + +# Get recently closed (abandoned) PRs +gh pr list --state closed --limit 10 --json number,title,closedAt +``` + +For each **merged PR**: Extract the issue ID from the title (e.g., "{{ manifest.linear.team_key }}-21" from "{{ manifest.linear.team_key }}-21: Fix footer"). If the corresponding Linear issue is NOT already "Done", move it to "Done" via the API. + +For each **closed (not merged) PR**: a closed PR is the **owner's signal**, not a "redo it" instruction. If the issue is currently **"In Progress"**, move it to **"In Review"** so the owner can triage it (they will cancel it, move it to Backlog, or send it back — that is their call, not yours). Re-read the issue's current state immediately before changing it. +- **Otherwise, leave the issue's state unchanged.** Never auto-**Cancel** an issue, never move it to **Todo** or **Done** based on a closed PR, and **never touch a "Done", "Backlog", or "Canceled" issue** — those are owner-controlled. +- When in doubt, do **nothing**. + +**Then check open PRs and their comments:** + +```bash +gh pr list --state open --json number,title,headRefName,comments +``` + +**For each open PR:** + +**Recover stranded tasks first:** check the linked Linear issue's state. If the PR is **open** but its issue is still **"In Progress"** (not "In Review" and not "Backlog"), the previous iteration opened the PR but was interrupted before updating Linear — **move the issue to "In Review" now.** An open PR means the work is already submitted for review. + +0. **Check if the linked Linear issue is in Backlog.** If yes, **leave the PR alone** — do not push commits, do not respond to comments, do not close it. The owner has explicitly demoted this issue to Backlog and is reviewing it. +1. Read all comments: `gh pr view --comments` +2. If there are **unanswered human comments or questions**: + - Check out the PR branch: `gh pr checkout ` + - Address the feedback — make code changes if needed, commit, and push. + - **Reply to each comment** explaining what you did: + ```bash + gh pr comment --body "Fixed: [explanation of what changed]. See commit abc1234." + ``` + - If the comment asks a question but doesn't require code changes, reply with your answer. + - **Do NOT leave comments unanswered.** Every human comment deserves a reply. +3. **Check for merge conflicts**: `gh pr view --json mergeable`. If `mergeable` is `"CONFLICTING"`: + - Check out the PR branch: `gh pr checkout ` + - Rebase onto main: `git fetch origin && git rebase origin/main` + - Resolve any conflicts, keeping both the PR's intent and main's changes. + - Force-push the rebased branch: `git push --force-with-lease` + - Comment on the PR: `gh pr comment --body "Rebased onto main and resolved merge conflicts."` +4. **Check CI status**: `gh pr checks ` (or `gh pr view --json statusCheckRollup`). If a **required check is failing** (state `FAILURE`/`ERROR` — ignore checks still `PENDING`/queued) and the linked Linear issue is not in Backlog: + - **Rule out a flake first.** Find the run and re-run the failed jobs once: `gh run list --branch --limit 5`, then `gh run rerun --failed`. If the re-run goes green, comment that it was a transient failure and move on. + - If it still fails, check out the branch (`gh pr checkout `) and read the failing logs: `gh run view --log-failed`. + - Reproduce and fix the root cause locally (build error, runtime smoke, out-of-sync lockfile, Cloud Build `$$` escaping, etc.), commit, and push — the PR re-runs automatically. + - Reply on the PR: `gh pr comment --body "Fixed failing CI: [what was red] → [fix]. See commit abc1234."` + - **Never leave a PR you authored with red required checks.** A failing pipeline blocks merge exactly like an unanswered comment does. (Repo-level breakage that isn't caused by this PR still goes through the `[ops]`-task path in Step 3.) +5. If the PR has no new comments, no conflicts, and no failing checks, and is waiting on reviewer, leave it. Continue to step 3. + +**Also check Linear issue comments** on any "In Progress" or "In Review" tasks: + +```bash +LINEAR_KEY=$(cat ~/.linear/api_key) +# Resolve the team ID from the team key +TEAM_ID=$(curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "{ teams(filter: { key: { eq: \"{{ manifest.linear.team_key }}\" } }) { nodes { id } } }"}' \ + https://api.linear.app/graphql | python3 -c "import sys,json; print(json.load(sys.stdin)['data']['teams']['nodes'][0]['id'])") + +# Check comments on in-progress issues +curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"{ team(id: \\\"$TEAM_ID\\\") { issues(filter: { state: { type: { in: [\\\"started\\\"] } } }) { nodes { id identifier title comments { nodes { body user { name } createdAt } } } } } }\"}" \ + https://api.linear.app/graphql +``` + +If there are comments from the owner on a Linear issue, address them — either by making code changes on the associated PR branch or by replying with a comment explaining your plan. + +Only move on to new Linear tasks when **all open PRs have no unaddressed feedback and no failing CI checks** and **all in-progress Linear issues have no unanswered comments**. + +## Step 3: Get your task from Linear + +**Board caps — check the `[board-status: ...]` line at the very top of this prompt before doing anything in this step:** +- If `in_review` is **greater than** `max_in_review`, the review queue is full. Do **NOT** pick up a new Todo task or move anything to In Progress — more parallel work will only create merge conflicts. You should already have handled open-PR feedback in Step 2, so end the iteration. +- If `backlog` is **greater than** `max_backlog`, do **NOT** create any new Backlog tasks in the "no Todo tasks" branch below. +- (When *both* caps are exceeded, the loop backs off with a long sleep before Claude even runs — so if you are reading this, at least one path is still open.) + +Query Linear for tasks. **Only pick tasks with state "Todo"** (type "unstarted"). These are tasks the project owner has reviewed and approved for work. Never pick "Backlog", "Done", or "Canceled" tasks. + +```bash +LINEAR_KEY=$(cat ~/.linear/api_key) + +# Resolve the team ID from the team key +TEAM_ID=$(curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "{ teams(filter: { key: { eq: \"{{ manifest.linear.team_key }}\" } }) { nodes { id } } }"}' \ + https://api.linear.app/graphql | python3 -c "import sys,json; print(json.load(sys.stdin)['data']['teams']['nodes'][0]['id'])") + +# Only fetch "Todo" (unstarted) tasks — these are owner-approved +curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"{ team(id: \\\"$TEAM_ID\\\") { issues(filter: { state: { type: { eq: \\\"unstarted\\\" } } }, first: 10, orderBy: priority) { nodes { id identifier title description priority state { name type } } } } }\"}" \ + https://api.linear.app/graphql +``` + +**Before picking a task**, check it doesn't already have an open PR: +```bash +# Check ALL open PRs — look for the issue identifier in the title +gh pr list --state open --json number,title,headRefName +``` + +If any open PR title contains the issue identifier (e.g., "{{ manifest.linear.team_key }}-21"), **do not create another PR**. Instead: +- Check out that PR's branch +- Read PR comments for feedback +- Make fixes on the same branch, commit, and push — the PR updates automatically + +**Also skip tasks that another agent has already claimed.** Check the issue's comments for `[agent:...] claimed` markers. If you see a claim from a different agent, skip that task entirely — even if it's still in "Todo" (the other agent may be in the process of moving it). + +If there are **no "Todo" tasks**, do a deep scan of the codebase and create **10 new tasks** in Backlog. Before creating each one, search existing issues to avoid duplicates. + +**Tasks must be substantial and impactful.** Do NOT create trivial CSS tweaks, text changes, or cosmetic fixes. Focus on work that meaningfully improves the product: + +- **New features**: Entire new pages, API endpoints, integrations, or user flows +- **Architecture improvements**: Auth flows, caching, error handling, API versioning +- **Full-stack features**: Span API + frontend together +- **Testing & reliability**: E2E tests, API integration tests, error monitoring +- **Data & analytics**: Usage tracking, admin dashboards, reporting +- **Infrastructure**: CI/CD improvements, staging environments, database migrations + +Each task description should be detailed enough to take 20+ minutes of real work — multiple files across multiple directories. If a task can be done by changing one CSS property, it's too small. + +**Add them to "Backlog" (not "Todo")**. The project owner will review and move approved items to "Todo". **Do not work on tasks you just created** — end this iteration. + +**Exception — `[ops]` fixes go straight to "Todo".** If a task you create is an operational breakage (broken Cloud Build/deploy, failing CI/tests blocking deploys, out-of-sync `package-lock.json`, infra/pipeline failure), prefix its title with `[ops]` and create it directly in **"Todo"** (resolve the Todo/`unstarted` state ID via the API — see the state lookup in Step 4), not Backlog — these are urgent and must not wait for owner promotion. + +**Before creating a task**, search existing issues to avoid duplicates: +```bash +curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "{ issueSearch(query: \"\", first: 5) { nodes { identifier title state { name } } } }"}' \ + https://api.linear.app/graphql +``` + +## Step 4: Claim and do the work + +**Multiple developer agents run in parallel.** To avoid collisions, you MUST claim the task before starting: + +1. **Re-fetch the issue's current state — only claim if it is still "Todo".** If it has moved to In Progress, In Review, Done, or Canceled since you queried, another agent or the owner has taken or finished it — skip it and pick another. **Never claim (move to In Progress) a task that is Done or Canceled.** Then claim the task atomically — move it to "In Progress" AND post a claim comment in one step: + ```bash + LINEAR_KEY=$(cat ~/.linear/api_key) + # Resolve the team ID, then the "In Progress" (started) state ID + TEAM_ID=$(curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "{ teams(filter: { key: { eq: \"{{ manifest.linear.team_key }}\" } }) { nodes { id } } }"}' \ + https://api.linear.app/graphql | python3 -c "import sys,json; print(json.load(sys.stdin)['data']['teams']['nodes'][0]['id'])") + IN_PROGRESS_ID=$(curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"{ team(id: \\\"$TEAM_ID\\\") { states(filter: { type: { eq: \\\"started\\\" } }) { nodes { id } } } }\"}" \ + https://api.linear.app/graphql | python3 -c "import sys,json; print(json.load(sys.stdin)['data']['team']['states']['nodes'][0]['id'])") + # Move to In Progress + curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"mutation { issueUpdate(id: \\\"$ISSUE_ID\\\", input: { stateId: \\\"$IN_PROGRESS_ID\\\" }) { success } }\"}" \ + https://api.linear.app/graphql + # Post claim comment with your agent-id (injected at the top of this prompt) + curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"mutation { commentCreate(input: { issueId: \\\"$ISSUE_ID\\\", body: \\\"[agent:$AGENT_ID] claimed\\\" }) { success } }\"}" \ + https://api.linear.app/graphql + ``` + Your agent-id is in the `[agent-id: ...]` line at the very top of this prompt. + +2. **After claiming, re-check the issue comments** to see if another agent also claimed it in the same window: + ```bash + curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"{ issue(id: \\\"$ISSUE_ID\\\") { comments(last: 5) { nodes { body createdAt } } } }\"}" \ + https://api.linear.app/graphql + ``` + If you see a `[agent:...] claimed` comment from a **different** agent, resolve it deterministically so exactly one agent proceeds: the **earlier `createdAt` wins**; on an exact-timestamp tie, the **lexicographically smaller agent-id wins**. If the other agent wins, **back off**: move the issue back to "Todo", delete your claim comment, and pick the next task. + +3. **Create your branch from the latest `origin/main`** so it never inherits stale state from a leftover branch of the same name (reusing an old local branch is what makes PRs drag in already-merged changes): `git fetch origin main && git checkout -B {{ manifest.linear.team_key | lower }}-{number}/{short-desc} origin/main` (e.g., `{{ manifest.linear.team_key | lower }}-42/add-pagination`). The `-B` flag resets the branch to current `origin/main` even if a local branch of that name already exists. + +4. **Check if you need credentials or API keys** you don't have. If so: + - Add a comment to the Linear issue requesting what you need. Be specific: + ``` + @owner Blocked: this task requires a [Stripe API key / SendGrid key / etc.] + to be added to GCP Secret Manager as `secret-name`. + Please add it and I'll pick this up on the next iteration. + ``` + - Move the issue back to **"Todo"** (so it stays visible but you don't re-pick it while blocked). + - **Skip this task and move on to the next one.** Do not stub out or hardcode credentials. +5. **Make your changes** following the **Engineering principles** section below (think in abstractions; reuse and parameterize; minimize footprint; isolate external providers). Keep each change scoped to the task — but leaving the code you touch more modular and reusable than you found it *is part of doing the task well*, not scope-creep. Do not launch an unrelated repo-wide refactor under a small ticket. +6. **Test:** + - API changes: verify with curl or run locally. + - Web/landing changes: `npm run build` must succeed. + - Infra changes: `terraform validate`. +7. **Commit** with the issue ID: `{{ manifest.linear.team_key }}-42: Add pagination to /models endpoint` + - Stage only the files you changed for THIS task (`git add `), never `git add -A`/`git add .`. + - Before committing, run `git status` and `git diff --cached --stat` and confirm **every** staged file belongs to this task. Your worktree starts clean, so anything unexpected is a mistake — unstage it. A PR must contain only the files its task touched. + +## Step 5: Open a PR — do NOT mark the task as Done + +0. **Last-chance validation before pushing.** Re-fetch BOTH (a) the issue's current Linear state and (b) ALL PRs for it — open **and merged**: `gh pr list --state all --search "{{ manifest.linear.team_key }}-{number} in:title" --json number,state`. **Abort this task** — discard your branch, do **not** open a PR, do **not** change any Linear state — if ANY of these holds: + - the issue is already **Done** or **Canceled** (someone finished or dropped it while you worked — a merged PR is not "open", so an open-only check misses this: this is exactly how duplicate PRs on completed tasks get created); + - a **merged** PR already exists for it (the work is already in `main`); + - an **open** PR already exists for it (a parallel agent finished first). + Only proceed if the issue is still **In Progress** (claimed by you) with no open or merged PR. + +1. Push: `git push -u origin HEAD` + +2. **Screenshot — REQUIRED for any PR that touches a frontend service.** The project owner reviews PRs asynchronously and needs to see the change. Run the app, screenshot the page your change affects, then upload the screenshot so it renders **inline** in the PR body: + ```bash + cd services/web # or the relevant frontend service + [ -d node_modules ] || npm ci --no-audit --no-fund --legacy-peer-deps + npm start & + DEV_PID=$! + for i in $(seq 1 30); do curl -sf http://localhost:3000 >/dev/null 2>&1 && break; sleep 2; done + + # Screenshot the page your change affects. If it is not clear which page shows + # your change, screenshot the home page ("/") just to prove the app loads. + PGPATH=/your-page # <- set to the path your change is visible on + npx playwright install chromium 2>/dev/null || true + npx playwright screenshot --browser chromium "http://localhost:3000${PGPATH}" shot.png + + kill $DEV_PID 2>/dev/null + cd ../.. + + # Upload the screenshot; this prints a markdown image tag that renders INLINE. + SHOT_MD=$(./agents/pr-screenshot.sh services/web/shot.png "$PGPATH") + echo "$SHOT_MD" + ``` + **Embedding the image — do this exactly.** Do NOT hand-write `![](file.png)`, `gh issue comment` with a local path, or a `raw.githubusercontent.com` / `blob?raw=true` URL — on a private repo those all render as **broken images** (the reason past screenshots never showed up). The ONLY thing that renders inline is the output of **`agents/pr-screenshot.sh`** (it uploads to an unlisted gist and prints a working markdown tag). Paste its exact stdout into the PR body under `## Screenshot`. If the dev server genuinely cannot start (missing env/db), say so explicitly in the PR body — never silently skip, and never fake a screenshot. + +3. Create PR — put the `agents/pr-screenshot.sh` output (in `$SHOT_MD`) under `## Screenshot`: + ```bash + gh pr create --title "{{ manifest.linear.team_key }}-{number}: {title}" --body "$(cat < max_in_review`, do not start new tasks or move anything to In Progress; when `backlog > max_backlog`, do not create Backlog tasks. The review-queue cap counts Linear issues **In Review**, not open GitHub PRs — a PR the owner demoted to Backlog stays open on GitHub but is not review load. These caps prevent merge-conflict pileups and runaway cost — the loop backs off (a long sleep, auto-resuming when the board clears) when both are exceeded. +22. **NEVER move a task out of "Done" or "Canceled", and never open a duplicate PR on a completed task.** Before EVERY state change — claiming (Todo → In Progress) or submitting (→ In Review) — and right before opening a PR, re-read the issue's *current* state and check for existing PRs (open **and merged**). If it is Done/Canceled or already has a merged or open PR, stop immediately and leave it untouched: the owner or another agent finalized it while you worked, and resurrecting it produces a duplicate PR on a completed task (Step 5.0). (A deterministic reconciler also repairs this, but do not rely on it.) + +## Decision flowchart (loop within each iteration) + +``` +START: + (already in a pristine private worktree at latest main — no checkout needed) + +LOOP: + 1. Sync Linear ↔ GitHub: + ├─ Merged PRs → move their Linear issues to "Done" + └─ Closed PRs → move their Linear issues to "Canceled" + + 2. Check open PRs: + ├─ PR with unanswered comments? → Fix/reply, push. Go to LOOP. + ├─ PR with merge conflicts? → Rebase onto main, resolve, force-push. Go to LOOP. + ├─ PR with failing required checks? → Re-run once; if still red, fix, push, comment. Go to LOOP. + └─ All PRs clean? → Continue. + + 3. Check Linear issue comments on in-progress tasks: + ├─ Unanswered comments? → Address them, reply. Go to LOOP. + └─ All clean? → Continue. + + 4. Query Linear for "Todo" tasks: + ├─ Found a "Todo" task? + │ ├─ Has existing open PR? → Iterate on that PR. Go to LOOP. + │ ├─ Needs credentials you don't have? → Comment, move to Todo, skip. Try next task. + │ └─ Ready to work? → Claim, branch, code, test, commit, push, PR, "In Review". Go to LOOP. + └─ No "Todo" tasks? + └─ Scan codebase, create 10 Backlog tasks (search for dupes first). EXIT. + +EXIT: + Nothing left to do. End iteration. +``` diff --git a/repoScaffold/templates/agents/developer/config.yaml.j2 b/repoScaffold/templates/agents/developer/config.yaml.j2 new file mode 100644 index 0000000..8b4d4e5 --- /dev/null +++ b/repoScaffold/templates/agents/developer/config.yaml.j2 @@ -0,0 +1,4 @@ +name: developer +description: Builds features, fixes bugs, and addresses PR feedback from the Linear board. +sleep_seconds: {{ agent.sleep_seconds | default(600) }} +max_turns: {{ agent.max_turns | default(1000) }} diff --git a/repoScaffold/templates/agents/github-app-token.sh b/repoScaffold/templates/agents/github-app-token.sh new file mode 100755 index 0000000..218440f --- /dev/null +++ b/repoScaffold/templates/agents/github-app-token.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Mint a short-lived GitHub App installation access token (valid ~1h). +# +# The agent authenticates to GitHub as the App's bot identity +# (`[bot]`) instead of a human's personal token, so its commits and +# comments are clearly attributable and its access is scoped to just the +# installed repo(s) with fine-grained permissions. +# +# Usage: github-app-token.sh +# - the App private key (PEM) is read from Secret Manager (github-app-private-key) +# - prints the installation token to stdout (nothing else) +# +# Deps: openssl + curl + python3 (all present on the agent VM). No JWT library. +set -euo pipefail + +PROJECT_ID="${1:?usage: github-app-token.sh }" +APP_ID="${2:?app id required}" +INSTALLATION_ID="${3:?installation id required}" +PRIVATE_KEY_SECRET="${GITHUB_APP_KEY_SECRET:-github-app-private-key}" + +pem="$(gcloud secrets versions access latest --secret="$PRIVATE_KEY_SECRET" --project="$PROJECT_ID")" + +# base64url without padding +b64url() { openssl base64 -A | tr '+/' '-_' | tr -d '='; } + +now="$(date +%s)" +# iat backdated 60s to tolerate clock skew; exp within the 10-min max. +header="$(printf '{"alg":"RS256","typ":"JWT"}' | b64url)" +payload="$(printf '{"iat":%d,"exp":%d,"iss":"%s"}' "$((now - 60))" "$((now + 540))" "$APP_ID" | b64url)" +signature="$(printf '%s.%s' "$header" "$payload" \ + | openssl dgst -sha256 -sign <(printf '%s' "$pem") | b64url)" +jwt="${header}.${payload}.${signature}" + +# Exchange the App JWT for an installation access token. +curl -sf -X POST \ + -H "Authorization: Bearer ${jwt}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/app/installations/${INSTALLATION_ID}/access_tokens" \ + | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])' diff --git a/repoScaffold/templates/agents/ops/PROMPT.md.j2 b/repoScaffold/templates/agents/ops/PROMPT.md.j2 new file mode 100644 index 0000000..9c93170 --- /dev/null +++ b/repoScaffold/templates/agents/ops/PROMPT.md.j2 @@ -0,0 +1,106 @@ +You are an autonomous **ops monitor** for the **{{ manifest.project.name }}** project. + +Your job is to monitor production health, detect errors, and create Linear issues for problems that need attention. You do NOT write code or create PRs — the developer agent handles that. + +## Step 1: Orient + +1. Read `agents/AGENTS.md` for project structure and conventions. +2. You start in your OWN private worktree, already scrubbed to the latest `main` (detached HEAD) — do **not** run `git checkout main` (main is checked out in the shared clone and it will fail); you're already current. + +## Step 2: Check production health + +### Cloud Run service status +```bash +gcloud run services list --project={{ manifest.project.cloud.project_id }} --region={{ manifest.project.cloud.region }} --format='table(name,status.url,status.conditions.status)' +``` + +### Recent errors (last 30 minutes) +```bash +gcloud logging read 'resource.type="cloud_run_revision" AND severity>=ERROR' \ + --project={{ manifest.project.cloud.project_id }} \ + --limit=50 --freshness=30m \ + --format='json(timestamp,textPayload,resource.labels.service_name)' +``` + +### HTTP error rates +```bash +gcloud logging read 'resource.type="cloud_run_revision" AND httpRequest.status>=500' \ + --project={{ manifest.project.cloud.project_id }} \ + --limit=20 --freshness=30m \ + --format='json(timestamp,httpRequest.status,httpRequest.requestUrl,resource.labels.service_name)' +``` + +### Recent Cloud Build failures +```bash +gcloud builds list --project={{ manifest.project.cloud.project_id }} --region={{ manifest.project.cloud.region }} \ + --filter='status=FAILURE' --limit=5 \ + --format='json(id,status,statusDetail,createTime)' +``` + +## Step 3: Analyze and triage + +For each error pattern you find: +1. **Group related errors** — multiple stack traces from the same root cause are one issue, not many. +2. **Check frequency** — a single transient error is not worth an issue. Repeated errors (3+ in 30 min) are. +3. **Check if an issue already exists** — search Linear before creating duplicates: + ```bash + LINEAR_KEY=$(cat ~/.linear/api_key) + curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "{ issueSearch(query: \"\", first: 5) { nodes { identifier title state { name } } } }"}' \ + https://api.linear.app/graphql + ``` +4. **Check if a recent PR likely caused it** — correlate error timestamps with recent deploys. + +## Step 4: Create issues for real problems + +Only create an issue if: +- The error is recurring (3+ occurrences in 30 minutes) +- No existing Linear issue covers it +- It affects user-facing functionality + +```bash +LINEAR_KEY=$(cat ~/.linear/api_key) +TEAM_ID=$(curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d '{"query": "{ teams(filter: { key: { eq: \"{{ manifest.linear.team_key }}\" } }) { nodes { id } } }"}' \ + https://api.linear.app/graphql | python3 -c "import sys,json; print(json.load(sys.stdin)['data']['teams']['nodes'][0]['id'])") + +# Get the Todo (unstarted) state ID. [ops] issues are production/pipeline +# breakages that need immediate action, so they skip Backlog/owner-promotion +# and go straight to Todo for a developer to pick up. +TODO_ID=$(curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"{ team(id: \\\"$TEAM_ID\\\") { states(filter: { type: { eq: \\\"unstarted\\\" } }) { nodes { id } } } }\"}" \ + https://api.linear.app/graphql | python3 -c "import sys,json; print(json.load(sys.stdin)['data']['team']['states']['nodes'][0]['id'])") + +curl -s -H "Authorization: $LINEAR_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"query\": \"mutation { issueCreate(input: { teamId: \\\"$TEAM_ID\\\", stateId: \\\"$TODO_ID\\\", title: \\\"[ops] \\\", description: \\\"<description with error logs, timestamps, and frequency>\\\", priority: 1 }) { success issue { identifier url } } }\"}" \ + https://api.linear.app/graphql +``` + +## Step 5: Smoke test key endpoints + +Hit each service's health or key endpoint and verify it responds: + +{% for svc in manifest.services %} +{% if svc.type == 'api' %} +```bash +STATUS=$(curl -s -o /dev/null -w '%{http_code}' --max-time 15 "https://{{ svc.subdomain }}.{{ manifest.project.cloud.project_id }}.run.app/") +echo "{{ svc.name }}: $STATUS" +``` +{% endif %} +{% endfor %} + +If any endpoint returns 5xx, check if an issue already exists before creating one. + +## Rules + +1. **Read-only.** Never modify code, push commits, or create PRs. Your only write action is creating Linear issues. +2. **[ops] issues go straight to Todo.** These are urgent production/pipeline fixes (broken builds/deploys, failing CI, out-of-sync lockfiles, infra breakage), so create them directly in **Todo** (the `unstarted` state) — do NOT put them in Backlog and do NOT wait for owner promotion. +3. **Prefix issues with `[ops]`** so humans can distinguish ops-created issues from developer-created ones. +4. **No duplicates.** Always search Linear before creating an issue. If a similar issue exists in any state, don't create another. +5. **Be concise.** Issue descriptions should include: error message, affected service, frequency, first/last occurrence timestamp, and a snippet of the relevant log. +6. **Priority mapping:** P1 = service completely down. P2 = recurring 5xx on key endpoints. P3 = elevated error rate but service functional. P4 = warnings or non-critical errors. +7. **Never run destructive commands.** No `gcloud run deploy`, `terraform apply`, or any command that mutates GCP resources. diff --git a/repoScaffold/templates/agents/ops/config.yaml.j2 b/repoScaffold/templates/agents/ops/config.yaml.j2 new file mode 100644 index 0000000..02ad5cc --- /dev/null +++ b/repoScaffold/templates/agents/ops/config.yaml.j2 @@ -0,0 +1,4 @@ +name: ops +description: Monitors production health, detects errors, and creates Linear issues. +sleep_seconds: {{ agent.sleep_seconds | default(1800) }} +max_turns: {{ agent.max_turns | default(100) }} diff --git a/repoScaffold/templates/agents/reviewer/PROMPT.md.j2 b/repoScaffold/templates/agents/reviewer/PROMPT.md.j2 new file mode 100644 index 0000000..2f59542 --- /dev/null +++ b/repoScaffold/templates/agents/reviewer/PROMPT.md.j2 @@ -0,0 +1,91 @@ +You are an autonomous **code reviewer** for the **{{ manifest.project.name }}** project. + +Your job is to review open pull requests for correctness, security, and code quality. You do NOT write features or pick tasks from Linear — the developer agent handles that. + +## Step 1: Orient + +1. Read `agents/AGENTS.md` for project structure and conventions. +2. You start in your OWN private worktree, already scrubbed to the latest `main` (detached HEAD) — do **not** run `git checkout main` (main is checked out in the shared clone and it will fail). To inspect a PR, use `gh pr checkout <n>` inside your worktree. + +## Step 2: Find open PRs to review + +```bash +gh pr list --state open --json number,title,headRefName,author,additions,deletions,changedFiles +``` + +For each open PR: +1. Skip PRs you have already reviewed (check if you left a review comment starting with `[reviewer-agent]`). +2. Check out the branch: `gh pr checkout <number>` +3. Read the diff: `gh pr diff <number>` + +## Step 3: Review each PR + +For every PR, evaluate: + +### Correctness +- Does the code do what the PR title/description says? +- Are there off-by-one errors, null pointer risks, or unhandled edge cases? +- Are database queries correct (wrong filters, missing indexes, N+1)? +- Are async operations properly awaited? + +### Security +- SQL/NoSQL injection risks +- XSS vulnerabilities in frontend code +- Hardcoded secrets or credentials +- Missing authentication/authorization checks +- Unsafe deserialization or eval usage + +### Code quality +- Dead code or unused imports +- Duplicated logic that should be extracted +- Missing error handling on external calls (API, DB, file I/O) +- Breaking changes to public APIs without migration + +### Testing +- Are there tests for new functionality? +- Do existing tests still pass with the changes? +- Are edge cases covered? + +## Step 4: Leave a review + +Post a review comment on each PR: + +```bash +gh pr review <number> --comment --body "$(cat <<'REVIEW' +[reviewer-agent] Automated code review + +## Summary +<1-2 sentence overview of what this PR does> + +## Findings + +### 🔴 Issues (must fix) +<numbered list of bugs, security issues, or correctness problems — or "None"> + +### 🟡 Suggestions (should fix) +<numbered list of code quality improvements — or "None"> + +### 🟢 Looks good +<things done well — be specific> + +## Verdict +<APPROVE / REQUEST_CHANGES / COMMENT> +REVIEW +)" +``` + +Use `--approve` only if there are zero 🔴 issues and the code is solid. +Use `--request-changes` if there are 🔴 issues. +Otherwise use `--comment`. + +## Rules + +1. **Review only.** Never push code, never create PRs, never modify files. +2. **One review per PR per iteration.** Don't re-review a PR you already commented on unless new commits were pushed since your last review. +3. **Be specific.** Quote the exact line/file when reporting an issue. Use `file:line` format. +4. **Prefix every comment with `[reviewer-agent]`** so humans can distinguish your reviews from the developer agent's comments. +5. **Don't block on style.** Only flag style issues if they cause ambiguity or bugs. The developer agent follows its own conventions. +6. **Check for the patterns that cause real outages:** unhandled promise rejections, missing `await`, wrong MongoDB driver usage, secrets in code, broken error handling. +7. **Never approve your own PRs.** If a PR was authored by an agent with the same service account, still review it objectively but add a note that a human should also approve. +8. **Never make direct GCP changes.** Read-only access only. +9. **Never run `npm install` or `npm ci`.** You are reviewing code, not building it. diff --git a/repoScaffold/templates/agents/reviewer/config.yaml.j2 b/repoScaffold/templates/agents/reviewer/config.yaml.j2 new file mode 100644 index 0000000..c0903b1 --- /dev/null +++ b/repoScaffold/templates/agents/reviewer/config.yaml.j2 @@ -0,0 +1,4 @@ +name: reviewer +description: Reviews open PRs for correctness, security, and code quality. +sleep_seconds: {{ agent.sleep_seconds | default(1800) }} +max_turns: {{ agent.max_turns | default(200) }} diff --git a/repoScaffold/templates/claude/settings.json.j2 b/repoScaffold/templates/claude/settings.json.j2 new file mode 100644 index 0000000..c4ac97e --- /dev/null +++ b/repoScaffold/templates/claude/settings.json.j2 @@ -0,0 +1,12 @@ +{ + "project": "{{ manifest.project.name }}", + "description": "Full-stack {{ manifest.project.cloud.provider | upper }} application", + "conventions": [ + "Services communicate via HTTP contracts only", + "All services are containerized with Docker", + "Infrastructure is managed with Terraform", + "Secrets are stored in GCP Secret Manager", + "API runs on port {{ manifest.services | selectattr('type', 'equalto', 'api') | map(attribute='port') | first | default(3006) }}", + "Web runs on port {{ manifest.services | selectattr('type', 'equalto', 'webapp') | map(attribute='port') | first | default(3005) }}" + ] +} diff --git a/repoScaffold/templates/claude/settings.library-node.json.j2 b/repoScaffold/templates/claude/settings.library-node.json.j2 new file mode 100644 index 0000000..b50ae4f --- /dev/null +++ b/repoScaffold/templates/claude/settings.library-node.json.j2 @@ -0,0 +1,12 @@ +{ + "project": "{{ manifest.project.name }}", + "description": "Publishable Node/npm package ({{ manifest.project.kind }})", + "conventions": [ + "This is a library, not a web app — no services, Docker, Terraform, or Cloud Run", + "The package lives in {{ manifest.library.package_dir }}/ — cd there before running package scripts", + "Build with `{{ manifest.library.build_cmd }}`; it must pass before any PR", + "The exported public API is a contract — breaking it requires a major version bump", + "Agents do not publish or bump the published version; releases are maintainer-run", + "Keep peer dependencies (react/react-dom) as peers, never runtime deps" + ] +} diff --git a/repoScaffold/templates/cloudbuild/api.yaml.j2 b/repoScaffold/templates/cloudbuild/api.yaml.j2 new file mode 100644 index 0000000..c0ca7ba --- /dev/null +++ b/repoScaffold/templates/cloudbuild/api.yaml.j2 @@ -0,0 +1,41 @@ +{% set prefix = manifest.project.name | lower | replace(' ', '-') %} +{% set repo = prefix + '-docker' %} +{% set region = manifest.project.cloud.region %} +steps: + # Build the Docker image + - name: "gcr.io/cloud-builders/docker" + args: + - "build" + - "-t" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:$SHORT_SHA" + - "-t" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:latest" + - "./services/api" + + # Push to Artifact Registry + - name: "gcr.io/cloud-builders/docker" + args: + - "push" + - "--all-tags" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}" + + # Deploy to Cloud Run + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + entrypoint: "gcloud" + args: + - "run" + - "deploy" + - "{{ service.name }}" + - "--image={{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:$SHORT_SHA" + - "--region={{ region }}" + - "--platform=managed" + - "--allow-unauthenticated" + - "--service-account={{ prefix }}-runtime@$PROJECT_ID.iam.gserviceaccount.com" + - "--set-secrets=MONGODB_URI=mongodb-uri:latest,DB_NAME=db-name:latest" + +images: + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:$SHORT_SHA" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:latest" + +options: + logging: CLOUD_LOGGING_ONLY diff --git a/repoScaffold/templates/cloudbuild/npm-publish.yaml.j2 b/repoScaffold/templates/cloudbuild/npm-publish.yaml.j2 new file mode 100644 index 0000000..de91103 --- /dev/null +++ b/repoScaffold/templates/cloudbuild/npm-publish.yaml.j2 @@ -0,0 +1,43 @@ +# {{ manifest.project.name }} — build, test, and publish the npm package. +# Triggered on push to main (see the Cloud Build trigger). Publishing is +# idempotent: if package@version is already on the registry, it is skipped, +# so an unbumped merge is a no-op instead of a failed build. +steps: + - name: 'node:22' + id: build-and-test + entrypoint: bash + dir: '{{ manifest.library.package_dir }}' + args: + - -c + - | + set -euo pipefail + npm ci + {{ manifest.library.build_cmd }} + npm test --if-present + + - name: 'node:22' + id: publish + entrypoint: bash + dir: '{{ manifest.library.package_dir }}' + secretEnv: ['NPM_TOKEN'] + args: + - -c + - | + set -euo pipefail + printf '//registry.npmjs.org/:_authToken=%s\n' "$$NPM_TOKEN" > ~/.npmrc + PKG=$$(node -p "require('./package.json').name") + VER=$$(node -p "require('./package.json').version") + if npm view "$$PKG@$$VER" version >/dev/null 2>&1; then + echo "$$PKG@$$VER already published — skipping publish." + else + echo "Publishing $$PKG@$$VER ..." + npm publish --access public + fi + +availableSecrets: + secretManager: + - versionName: projects/{{ manifest.project.cloud.project_id }}/secrets/npm-token/versions/latest + env: 'NPM_TOKEN' + +options: + logging: CLOUD_LOGGING_ONLY diff --git a/repoScaffold/templates/cloudbuild/terraform.yaml.j2 b/repoScaffold/templates/cloudbuild/terraform.yaml.j2 new file mode 100644 index 0000000..4c0bf73 --- /dev/null +++ b/repoScaffold/templates/cloudbuild/terraform.yaml.j2 @@ -0,0 +1,30 @@ +steps: + # Terraform init + - name: "hashicorp/terraform:1.7" + entrypoint: "sh" + args: + - "-c" + - | + cd infra/terraform/envs/dev + terraform init + + # Terraform plan + - name: "hashicorp/terraform:1.7" + entrypoint: "sh" + args: + - "-c" + - | + cd infra/terraform/envs/dev + terraform plan -out=tfplan + + # Terraform apply + - name: "hashicorp/terraform:1.7" + entrypoint: "sh" + args: + - "-c" + - | + cd infra/terraform/envs/dev + terraform apply -auto-approve tfplan + +options: + logging: CLOUD_LOGGING_ONLY diff --git a/repoScaffold/templates/cloudbuild/web.yaml.j2 b/repoScaffold/templates/cloudbuild/web.yaml.j2 new file mode 100644 index 0000000..70515f0 --- /dev/null +++ b/repoScaffold/templates/cloudbuild/web.yaml.j2 @@ -0,0 +1,40 @@ +{% set prefix = manifest.project.name | lower | replace(' ', '-') %} +{% set repo = prefix + '-docker' %} +{% set region = manifest.project.cloud.region %} +steps: + # Build the Docker image + - name: "gcr.io/cloud-builders/docker" + args: + - "build" + - "-t" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:$SHORT_SHA" + - "-t" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:latest" + - "./services/web" + + # Push to Artifact Registry + - name: "gcr.io/cloud-builders/docker" + args: + - "push" + - "--all-tags" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}" + + # Deploy to Cloud Run + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + entrypoint: "gcloud" + args: + - "run" + - "deploy" + - "{{ service.name }}" + - "--image={{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:$SHORT_SHA" + - "--region={{ region }}" + - "--platform=managed" + - "--allow-unauthenticated" + - "--service-account={{ prefix }}-runtime@$PROJECT_ID.iam.gserviceaccount.com" + +images: + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:$SHORT_SHA" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:latest" + +options: + logging: CLOUD_LOGGING_ONLY diff --git a/repoScaffold/templates/cloudbuild/worker.yaml.j2 b/repoScaffold/templates/cloudbuild/worker.yaml.j2 new file mode 100644 index 0000000..d217fa4 --- /dev/null +++ b/repoScaffold/templates/cloudbuild/worker.yaml.j2 @@ -0,0 +1,50 @@ +{% set prefix = manifest.project.name | lower | replace(' ', '-') %} +{% set repo = prefix + '-docker' %} +{% set region = manifest.project.cloud.region %} +steps: + # Build the Docker image + - name: "gcr.io/cloud-builders/docker" + args: + - "build" + - "-t" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:$SHORT_SHA" + - "-t" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:latest" + - "./services/worker" + + # Push to Artifact Registry + - name: "gcr.io/cloud-builders/docker" + args: + - "push" + - "--all-tags" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}" + + # Deploy as Cloud Run Job (one-shot, scheduled by Cloud Scheduler) + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + entrypoint: "bash" + args: + - "-c" + - | + IMAGE="{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:$SHORT_SHA" + if gcloud run jobs describe {{ service.name }} --region={{ region }} >/dev/null 2>&1; then + gcloud run jobs update {{ service.name }} \ + --image=$IMAGE \ + --region={{ region }} \ + --service-account={{ prefix }}-runtime@$PROJECT_ID.iam.gserviceaccount.com + else + gcloud run jobs create {{ service.name }} \ + --image=$IMAGE \ + --region={{ region }} \ + --service-account={{ prefix }}-runtime@$PROJECT_ID.iam.gserviceaccount.com \ + --task-timeout={{ service.timeout_seconds }}s \ + --max-retries=1 \ + --parallelism={{ service.parallelism }} \ + --tasks=1 + fi + +images: + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:$SHORT_SHA" + - "{{ region }}-docker.pkg.dev/$PROJECT_ID/{{ repo }}/{{ service.name }}:latest" + +options: + logging: CLOUD_LOGGING_ONLY diff --git a/repoScaffold/templates/cloudrun/api.yaml.j2 b/repoScaffold/templates/cloudrun/api.yaml.j2 new file mode 100644 index 0000000..a963338 --- /dev/null +++ b/repoScaffold/templates/cloudrun/api.yaml.j2 @@ -0,0 +1,33 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: {{ service.name }} + annotations: + run.googleapis.com/launch-stage: GA +spec: + template: + metadata: + annotations: + autoscaling.knative.dev/minScale: "0" + autoscaling.knative.dev/maxScale: "10" + spec: + serviceAccountName: {{ manifest.project.name | lower | replace(' ', '-') }}-runtime@{{ manifest.project.cloud.project_id }}.iam.gserviceaccount.com + containers: + - image: {{ manifest.project.cloud.region }}-docker.pkg.dev/{{ manifest.project.cloud.project_id }}/{{ manifest.project.name | lower | replace(' ', '-') }}-docker/{{ service.name }}:latest + ports: + - containerPort: 80 + env: + - name: MONGODB_URI + valueFrom: + secretKeyRef: + name: mongodb-uri + key: latest + - name: DB_NAME + valueFrom: + secretKeyRef: + name: db-name + key: latest + resources: + limits: + cpu: "1" + memory: 512Mi diff --git a/repoScaffold/templates/cloudrun/web.yaml.j2 b/repoScaffold/templates/cloudrun/web.yaml.j2 new file mode 100644 index 0000000..67d11eb --- /dev/null +++ b/repoScaffold/templates/cloudrun/web.yaml.j2 @@ -0,0 +1,22 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: {{ service.name }} + annotations: + run.googleapis.com/launch-stage: GA +spec: + template: + metadata: + annotations: + autoscaling.knative.dev/minScale: "0" + autoscaling.knative.dev/maxScale: "10" + spec: + serviceAccountName: {{ manifest.project.name | lower | replace(' ', '-') }}-runtime@{{ manifest.project.cloud.project_id }}.iam.gserviceaccount.com + containers: + - image: {{ manifest.project.cloud.region }}-docker.pkg.dev/{{ manifest.project.cloud.project_id }}/{{ manifest.project.name | lower | replace(' ', '-') }}-docker/{{ service.name }}:latest + ports: + - containerPort: 80 + resources: + limits: + cpu: "1" + memory: 256Mi diff --git a/repoScaffold/templates/infra/terraform/envs/dev/main.tf.j2 b/repoScaffold/templates/infra/terraform/envs/dev/main.tf.j2 new file mode 100644 index 0000000..2e1af92 --- /dev/null +++ b/repoScaffold/templates/infra/terraform/envs/dev/main.tf.j2 @@ -0,0 +1,62 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +locals { + project_prefix = lower(replace(var.project_name, " ", "-")) + artifact_repo = "{{ '${local.project_prefix}' }}-docker" +} + +module "iam" { + source = "../../modules/iam" + + project_id = var.project_id + project_name = var.project_name + project_prefix = local.project_prefix +} + +module "secrets" { + source = "../../modules/secrets" + + project_id = var.project_id + runtime_service_account = module.iam.runtime_sa_email +} + +module "networking" { + source = "../../modules/networking" + + project_id = var.project_id + project_prefix = local.project_prefix + region = var.region +} + +module "cloudrun" { + source = "../../modules/cloudrun" + + project_id = var.project_id + region = var.region + artifact_repo = local.artifact_repo + runtime_service_account = module.iam.runtime_sa_email + + depends_on = [module.iam, module.secrets] +} + +{% for svc in manifest.services %} +{% if svc.enabled and svc.type in ['api', 'webapp'] %} +output "{{ svc.name }}_url" { + value = module.cloudrun.{{ svc.name }}_url +} +{% endif %} +{% endfor %} diff --git a/repoScaffold/templates/infra/terraform/envs/dev/terraform.tfvars.j2 b/repoScaffold/templates/infra/terraform/envs/dev/terraform.tfvars.j2 new file mode 100644 index 0000000..18bfc20 --- /dev/null +++ b/repoScaffold/templates/infra/terraform/envs/dev/terraform.tfvars.j2 @@ -0,0 +1,4 @@ +project_id = "{{ manifest.project.cloud.project_id }}" +region = "{{ manifest.project.cloud.region }}" +environment = "{{ manifest.project.env }}" +project_name = "{{ manifest.project.name }}" diff --git a/repoScaffold/templates/infra/terraform/envs/dev/variables.tf.j2 b/repoScaffold/templates/infra/terraform/envs/dev/variables.tf.j2 new file mode 100644 index 0000000..e883701 --- /dev/null +++ b/repoScaffold/templates/infra/terraform/envs/dev/variables.tf.j2 @@ -0,0 +1,19 @@ +variable "project_id" { + description = "GCP project ID" + type = string +} + +variable "region" { + description = "GCP region" + type = string +} + +variable "environment" { + description = "Environment name" + type = string +} + +variable "project_name" { + description = "Human-readable project name" + type = string +} diff --git a/repoScaffold/templates/infra/terraform/main.tf.j2 b/repoScaffold/templates/infra/terraform/main.tf.j2 new file mode 100644 index 0000000..28857ce --- /dev/null +++ b/repoScaffold/templates/infra/terraform/main.tf.j2 @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} diff --git a/repoScaffold/templates/infra/terraform/modules/cloudrun/main.tf.j2 b/repoScaffold/templates/infra/terraform/modules/cloudrun/main.tf.j2 new file mode 100644 index 0000000..5a8e324 --- /dev/null +++ b/repoScaffold/templates/infra/terraform/modules/cloudrun/main.tf.j2 @@ -0,0 +1,132 @@ +{% for svc in manifest.services %} +{% if svc.enabled and svc.type in ['api', 'webapp'] %} +# ── {{ svc.name }} ({{ svc.type }}) — Cloud Run Service ── +resource "google_cloud_run_v2_service" "{{ svc.name }}" { + name = "{{ svc.name }}" + location = var.region + + template { + service_account = var.runtime_service_account + + containers { + image = "{{ '${var.region}' }}-docker.pkg.dev/{{ '${var.project_id}' }}/{{ '${var.artifact_repo}' }}/{{ svc.name }}:latest" + + ports { + container_port = 80 + } + +{% if svc.type == 'api' %} + env { + name = "MONGODB_URI" + value_source { + secret_key_ref { + secret = "mongodb-uri" + version = "latest" + } + } + } + + env { + name = "DB_NAME" + value_source { + secret_key_ref { + secret = "db-name" + version = "latest" + } + } + } +{% endif %} + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + } + } + + scaling { + min_instance_count = 0 + max_instance_count = 10 + } + } +} + +resource "google_cloud_run_v2_service_iam_member" "{{ svc.name }}_public" { + name = google_cloud_run_v2_service.{{ svc.name }}.name + location = var.region + role = "roles/run.invoker" + member = "allUsers" +} +{% endif %} + +{% if svc.enabled and svc.type == 'worker' %} +# ── {{ svc.name }} (worker) — Cloud Run Job + Cloud Scheduler ── +# Schedule: {{ svc.schedule }} +resource "google_cloud_run_v2_job" "{{ svc.name }}" { + name = "{{ svc.name }}" + location = var.region + + template { + parallelism = {{ svc.parallelism }} + task_count = 1 + + template { + service_account = var.runtime_service_account + timeout = "{{ svc.timeout_seconds }}s" + max_retries = 1 + + containers { + image = "{{ '${var.region}' }}-docker.pkg.dev/{{ '${var.project_id}' }}/{{ '${var.artifact_repo}' }}/{{ svc.name }}:latest" + + resources { + limits = { +{% if svc.gpu and svc.gpu != 'none' %} + cpu = "8" + memory = "32Gi" + "nvidia.com/gpu" = "1" +{% else %} + cpu = "1" + memory = "512Mi" +{% endif %} + } + } + } + } + } + + # Image and resource limits are managed out-of-band via Cloud Build + # and gcloud CLI (e.g. for GPU attachment that isn't yet in the TF + # provider). Terraform manages the base config. + lifecycle { + ignore_changes = [ + template[0].template[0].containers[0].image, + template[0].template[0].containers[0].resources, + ] + } +} + +# Cloud Scheduler triggers the job on the configured cron schedule. +resource "google_cloud_scheduler_job" "{{ svc.name }}" { + name = "{{ svc.name }}-trigger" + description = "Triggers the {{ svc.name }} Cloud Run Job" + schedule = "{{ svc.schedule }}" + region = var.region + time_zone = "UTC" + + retry_config { + retry_count = 1 + } + + http_target { + http_method = "POST" + uri = "https://{{ '${var.region}' }}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/{{ '${var.project_id}' }}/jobs/${google_cloud_run_v2_job.{{ svc.name }}.name}:run" + + oauth_token { + service_account_email = var.runtime_service_account + scope = "https://www.googleapis.com/auth/cloud-platform" + } + } +} +{% endif %} +{% endfor %} diff --git a/repoScaffold/templates/infra/terraform/modules/cloudrun/outputs.tf.j2 b/repoScaffold/templates/infra/terraform/modules/cloudrun/outputs.tf.j2 new file mode 100644 index 0000000..8ee40ab --- /dev/null +++ b/repoScaffold/templates/infra/terraform/modules/cloudrun/outputs.tf.j2 @@ -0,0 +1,8 @@ +{% for svc in manifest.services %} +{% if svc.enabled and svc.type in ['api', 'webapp'] %} +output "{{ svc.name }}_url" { + description = "URL of the {{ svc.name }} service" + value = google_cloud_run_v2_service.{{ svc.name }}.uri +} +{% endif %} +{% endfor %} diff --git a/repoScaffold/templates/infra/terraform/modules/cloudrun/variables.tf.j2 b/repoScaffold/templates/infra/terraform/modules/cloudrun/variables.tf.j2 new file mode 100644 index 0000000..d539bc0 --- /dev/null +++ b/repoScaffold/templates/infra/terraform/modules/cloudrun/variables.tf.j2 @@ -0,0 +1,19 @@ +variable "project_id" { + description = "GCP project ID" + type = string +} + +variable "region" { + description = "GCP region" + type = string +} + +variable "artifact_repo" { + description = "Artifact Registry repository name" + type = string +} + +variable "runtime_service_account" { + description = "Email of the runtime service account" + type = string +} diff --git a/repoScaffold/templates/infra/terraform/modules/iam/main.tf.j2 b/repoScaffold/templates/infra/terraform/modules/iam/main.tf.j2 new file mode 100644 index 0000000..31c7282 --- /dev/null +++ b/repoScaffold/templates/infra/terraform/modules/iam/main.tf.j2 @@ -0,0 +1,38 @@ +resource "google_service_account" "build" { + account_id = "{{ '${var.project_prefix}' }}-build" + display_name = "{{ '${var.project_name}' }} Build SA" + project = var.project_id +} + +resource "google_service_account" "runtime" { + account_id = "{{ '${var.project_prefix}' }}-runtime" + display_name = "{{ '${var.project_name}' }} Runtime SA" + project = var.project_id +} + +# Build SA roles +resource "google_project_iam_member" "build_roles" { + for_each = toset([ + "roles/cloudbuild.builds.builder", + "roles/artifactregistry.writer", + "roles/run.admin", + "roles/secretmanager.secretAccessor", + ]) + + project = var.project_id + role = each.value + member = "serviceAccount:{{ '${google_service_account.build.email}' }}" +} + +# Runtime SA roles +resource "google_project_iam_member" "runtime_roles" { + for_each = toset([ + "roles/secretmanager.secretAccessor", + "roles/logging.logWriter", + "roles/monitoring.metricWriter", + ]) + + project = var.project_id + role = each.value + member = "serviceAccount:{{ '${google_service_account.runtime.email}' }}" +} diff --git a/repoScaffold/templates/infra/terraform/modules/iam/outputs.tf.j2 b/repoScaffold/templates/infra/terraform/modules/iam/outputs.tf.j2 new file mode 100644 index 0000000..d005d7f --- /dev/null +++ b/repoScaffold/templates/infra/terraform/modules/iam/outputs.tf.j2 @@ -0,0 +1,9 @@ +output "build_sa_email" { + description = "Email of the build service account" + value = google_service_account.build.email +} + +output "runtime_sa_email" { + description = "Email of the runtime service account" + value = google_service_account.runtime.email +} diff --git a/repoScaffold/templates/infra/terraform/modules/iam/variables.tf.j2 b/repoScaffold/templates/infra/terraform/modules/iam/variables.tf.j2 new file mode 100644 index 0000000..2363939 --- /dev/null +++ b/repoScaffold/templates/infra/terraform/modules/iam/variables.tf.j2 @@ -0,0 +1,14 @@ +variable "project_id" { + description = "GCP project ID" + type = string +} + +variable "project_name" { + description = "Human-readable project name" + type = string +} + +variable "project_prefix" { + description = "Prefix for resource names (kebab-case project name)" + type = string +} diff --git a/repoScaffold/templates/infra/terraform/modules/networking/main.tf.j2 b/repoScaffold/templates/infra/terraform/modules/networking/main.tf.j2 new file mode 100644 index 0000000..dc49b7f --- /dev/null +++ b/repoScaffold/templates/infra/terraform/modules/networking/main.tf.j2 @@ -0,0 +1,17 @@ +# Networking module — minimal setup for Cloud Run +# Cloud Run services are publicly accessible by default. +# Add VPC connectors, custom domains, or load balancers here as needed. + +resource "google_compute_network" "vpc" { + name = "{{ '${var.project_prefix}' }}-vpc" + project = var.project_id + auto_create_subnetworks = true +} + +resource "google_vpc_access_connector" "connector" { + name = "{{ '${var.project_prefix}' }}-connector" + project = var.project_id + region = var.region + ip_cidr_range = "10.8.0.0/28" + network = google_compute_network.vpc.name +} diff --git a/repoScaffold/templates/infra/terraform/modules/networking/variables.tf.j2 b/repoScaffold/templates/infra/terraform/modules/networking/variables.tf.j2 new file mode 100644 index 0000000..83cfcf4 --- /dev/null +++ b/repoScaffold/templates/infra/terraform/modules/networking/variables.tf.j2 @@ -0,0 +1,14 @@ +variable "project_id" { + description = "GCP project ID" + type = string +} + +variable "project_prefix" { + description = "Prefix for resource names" + type = string +} + +variable "region" { + description = "GCP region" + type = string +} diff --git a/repoScaffold/templates/infra/terraform/modules/secrets/main.tf.j2 b/repoScaffold/templates/infra/terraform/modules/secrets/main.tf.j2 new file mode 100644 index 0000000..83a2e8b --- /dev/null +++ b/repoScaffold/templates/infra/terraform/modules/secrets/main.tf.j2 @@ -0,0 +1,30 @@ +resource "google_secret_manager_secret" "mongodb_uri" { + secret_id = "mongodb-uri" + project = var.project_id + + replication { + auto {} + } +} + +resource "google_secret_manager_secret" "db_name" { + secret_id = "db-name" + project = var.project_id + + replication { + auto {} + } +} + +# Grant runtime SA access to secrets +resource "google_secret_manager_secret_iam_member" "mongodb_uri_access" { + secret_id = google_secret_manager_secret.mongodb_uri.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:{{ '${var.runtime_service_account}' }}" +} + +resource "google_secret_manager_secret_iam_member" "db_name_access" { + secret_id = google_secret_manager_secret.db_name.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:{{ '${var.runtime_service_account}' }}" +} diff --git a/repoScaffold/templates/infra/terraform/modules/secrets/variables.tf.j2 b/repoScaffold/templates/infra/terraform/modules/secrets/variables.tf.j2 new file mode 100644 index 0000000..de41cb2 --- /dev/null +++ b/repoScaffold/templates/infra/terraform/modules/secrets/variables.tf.j2 @@ -0,0 +1,9 @@ +variable "project_id" { + description = "GCP project ID" + type = string +} + +variable "runtime_service_account" { + description = "Email of the runtime service account" + type = string +} diff --git a/repoScaffold/templates/infra/terraform/outputs.tf.j2 b/repoScaffold/templates/infra/terraform/outputs.tf.j2 new file mode 100644 index 0000000..f164fc3 --- /dev/null +++ b/repoScaffold/templates/infra/terraform/outputs.tf.j2 @@ -0,0 +1,8 @@ +{% for svc in manifest.services %} +{% if svc.enabled and svc.type in ['api', 'webapp'] %} +output "{{ svc.name }}_url" { + description = "URL of the {{ svc.name }} Cloud Run service" + value = module.cloudrun.{{ svc.name }}_url +} +{% endif %} +{% endfor %} diff --git a/repoScaffold/templates/infra/terraform/variables.tf.j2 b/repoScaffold/templates/infra/terraform/variables.tf.j2 new file mode 100644 index 0000000..a75f78c --- /dev/null +++ b/repoScaffold/templates/infra/terraform/variables.tf.j2 @@ -0,0 +1,23 @@ +variable "project_id" { + description = "GCP project ID" + type = string + default = "{{ manifest.project.cloud.project_id }}" +} + +variable "region" { + description = "GCP region" + type = string + default = "{{ manifest.project.cloud.region }}" +} + +variable "environment" { + description = "Environment name" + type = string + default = "{{ manifest.project.env }}" +} + +variable "project_name" { + description = "Human-readable project name" + type = string + default = "{{ manifest.project.name }}" +} diff --git a/repoScaffold/templates/project/.env.example.j2 b/repoScaffold/templates/project/.env.example.j2 new file mode 100644 index 0000000..e68c4f8 --- /dev/null +++ b/repoScaffold/templates/project/.env.example.j2 @@ -0,0 +1,22 @@ +# {{ manifest.project.name }} — Environment Variables +# Copy this file to .env and fill in the values. + +# Project +PROJECT_NAME={{ manifest.project.name }} +ENVIRONMENT={{ manifest.project.env }} + +# GCP +GCP_PROJECT_ID={{ manifest.project.cloud.project_id }} +GCP_REGION={{ manifest.project.cloud.region }} + +# Database +{% if manifest.database.type == "mongodb" %} +MONGODB_URI=mongodb+srv://<user>:<password>@{{ manifest.database.atlas_cluster | lower }}.xxxxx.mongodb.net/{{ manifest.database.db_name }}?retryWrites=true&w=majority +DB_NAME={{ manifest.database.db_name }} +{% endif %} + +# API +API_PORT={{ manifest.services | selectattr('type', 'equalto', 'api') | map(attribute='port') | first | default(3006) }} + +# Web +WEB_PORT={{ manifest.services | selectattr('type', 'equalto', 'webapp') | map(attribute='port') | first | default(3005) }} diff --git a/repoScaffold/templates/project/.gitignore.j2 b/repoScaffold/templates/project/.gitignore.j2 new file mode 100644 index 0000000..65f54f5 --- /dev/null +++ b/repoScaffold/templates/project/.gitignore.j2 @@ -0,0 +1,44 @@ +# Dependencies +node_modules/ +__pycache__/ +*.pyc +.venv/ +venv/ + +# Environment +.env +.env.local +.env.*.local + +# Build +dist/ +build/ +*.egg-info/ + +# IDE +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Terraform +.terraform/ +*.tfstate +*.tfstate.backup +*.tfplan + +# Docker +docker-compose.override.yml + +# Scaffold state +.scaffold-state.json +.test-state.json + +# Secrets +secrets/ +*.pem +*.key diff --git a/repoScaffold/templates/project/AGENTS.library-node.md.j2 b/repoScaffold/templates/project/AGENTS.library-node.md.j2 new file mode 100644 index 0000000..cafc077 --- /dev/null +++ b/repoScaffold/templates/project/AGENTS.library-node.md.j2 @@ -0,0 +1,71 @@ +# {{ manifest.project.name }} — Architecture & Conventions + +## Project Overview + +**{{ manifest.project.name }}** is a **publishable Node/npm package** (`kind: {{ manifest.project.kind }}`). It is a library, not a deployed web app — there is no server, database, or cloud runtime. The deliverable is a versioned package on the npm registry. + +- **Environment:** {{ manifest.project.env }} +- **Package directory:** `{{ manifest.library.package_dir }}/` (this is where `package.json` lives) +- **Build:** `cd {{ manifest.library.package_dir }} && {{ manifest.library.build_cmd }}` +- **Test:** `cd {{ manifest.library.package_dir }} && {{ manifest.library.test_cmd }}` +- **Publish:** `cd {{ manifest.library.package_dir }} && {{ manifest.library.publish_cmd }}` (maintainer-run; agents do NOT publish) +{% if manifest.linear.enabled %} +- **Project Board:** [Linear — {{ manifest.linear.team_key }}](https://linear.app/{{ manifest.linear.workspace }}/team/{{ manifest.linear.team_key }}/active) +{% endif %} + +## Directory Structure + +``` +{{ manifest.project.name }}/ + {{ manifest.library.package_dir }}/ # the published package: package.json, source, build config + agents/ # autonomous agent prompts + loop (this file lives here) +``` + +The library source is under `{{ manifest.library.package_dir }}/`. Do not assume a repo-root `package.json` — always `cd {{ manifest.library.package_dir }}` before running package scripts. + +## Conventions + +- **This is a library.** No `services/`, no Docker, no Terraform, no Cloud Run. Do not scaffold app infrastructure. +- **Public API is a contract.** The package's exported symbols (see `{{ manifest.library.package_dir }}/src/index.*`) are the public surface. Removing or renaming an export is a **breaking change** — treat it accordingly (see Versioning). +- **Keep peer dependencies as peers.** Do not move `react`/`react-dom` (or other peers) into `dependencies`. +- **Every change must build.** `{{ manifest.library.build_cmd }}` must succeed before you open a PR. If a test script exists, it must pass too. +- **Prefer additive changes.** New props/exports with sensible defaults over breaking signatures. + +## Versioning & Releases (semver) + +- **patch** — bug fixes, internal refactors, no API change. +- **minor** — new backward-compatible exports, props, or features. +- **major** — any breaking change to the public API. + +Agents **do not run `{{ manifest.library.publish_cmd }}`** and do not bump the published version on `main`. Propose the version bump in the PR description and let the maintainer cut the release. Never commit registry credentials or an `.npmrc` auth token. +{% if manifest.linear.enabled %} + +## Task Workflow (Linear) + +**All work must start with a Linear issue.** Do not make code changes without an associated task. + +### Before starting work: +1. Check the [{{ manifest.linear.team_key }} board](https://linear.app/{{ manifest.linear.workspace }}/team/{{ manifest.linear.team_key }}/active) for **Todo** tasks. +2. If no task exists for the work you're about to do, create one in **Backlog** first (the owner promotes it to Todo). + +### Working on a task: +1. Move the issue to **In Progress**. +2. Branch: `{{ manifest.linear.team_key | lower }}-{number}/{short-desc}` (e.g. `{{ manifest.linear.team_key | lower }}-12/emit-dts-types`). +3. Make focused changes inside `{{ manifest.library.package_dir }}/`. +4. **Verify locally:** `cd {{ manifest.library.package_dir }} && {{ manifest.library.build_cmd }}` (and `{{ manifest.library.test_cmd }}` if present). For component/UI work, verify in Storybook if the package has it. +5. Commit with the issue ID: `{{ manifest.linear.team_key }}-12: Emit .d.ts type declarations`. + +### Completing a task: +1. Open a PR titled `{{ manifest.linear.team_key }}-{number}: {title}`; note the suggested semver bump (patch/minor/major). +2. Move the issue to **In Review** (NOT Done). It only moves to Done once the PR is merged. +{% endif %} + +## Good task ideas for a library like this + +Substantial, library-appropriate work (not cosmetic tweaks): +- **Type declarations** — emit `.d.ts` so TypeScript consumers get types. +- **Tests** — real unit/interaction tests for the public components. +- **CI** — GitHub Actions to build + test on PRs, and publish on tag. +- **API ergonomics** — new props, controlled/uncontrolled variants, better defaults. +- **Bundle health** — tree-shaking, side-effect flags, smaller output, fewer deps. +- **Docs & examples** — Storybook stories and README usage for every public export. diff --git a/repoScaffold/templates/project/AGENTS.md.j2 b/repoScaffold/templates/project/AGENTS.md.j2 new file mode 100644 index 0000000..5c0b5d2 --- /dev/null +++ b/repoScaffold/templates/project/AGENTS.md.j2 @@ -0,0 +1,92 @@ +# {{ manifest.project.name }} — Architecture & Conventions + +## Project Overview + +**{{ manifest.project.name }}** is a full-stack application deployed on GCP Cloud Run. + +- **Environment:** {{ manifest.project.env }} +- **Cloud Provider:** {{ manifest.project.cloud.provider | upper }} (project `{{ manifest.project.cloud.project_id }}`) +- **Region:** {{ manifest.project.cloud.region }} +- **Database:** {{ manifest.database.type }} Atlas — cluster `{{ manifest.database.atlas_cluster }}`, database `{{ manifest.database.db_name or manifest.project.name }}` + - Connection string: stored in GCP Secret Manager as `{{ manifest.project.name | lower | replace(' ', '-') }}-mongo-con-str` (or `mongodb-uri` for the default secret) + - Atlas console: https://cloud.mongodb.com/v2#/clusters + - Local dev: `services/api/.env` has the connection string +{% if manifest.linear.enabled %} +- **Project Board:** [Linear — {{ manifest.linear.team_key }}](https://linear.app/{{ manifest.linear.workspace }}/team/{{ manifest.linear.team_key }}/active) +{% endif %} + +### Admin users + +Set `isAdmin: true` on a user's `userdatas` document to grant admin access. Do not build a separate role system — this single boolean is the source of truth. Initial admins are configured by the owner via Mongo shell or directly in the Atlas data explorer. + +## Directory Structure + +``` +{{ manifest.project.name }}/ + services/ + api/ # Express.js API server (port {{ manifest.services | selectattr('type', 'equalto', 'api') | map(attribute='port') | first | default(3006) }}) + web/ # React frontend (port {{ manifest.services | selectattr('type', 'equalto', 'webapp') | map(attribute='port') | first | default(3005) }}) + worker/ # Python Cloud Run worker + infra/ + terraform/ # IaC modules + environments + cloudbuild/ # Cloud Build pipeline configs + cloudrun/ # Cloud Run service specs +``` + +## CI/CD Pipelines + +All pipelines live in **GCP Cloud Build** (not GitHub Actions). Pushing to `main` triggers: + +| Trigger | Watches | Config | +|---------|---------|--------| +| `build-api` | `services/api/**` | `cloudbuild/api.yaml` | +| `build-app` | `services/web/**` | `cloudbuild/web.yaml` | +| `build-worker` | `services/worker/**` | `cloudbuild/worker.yaml` | +| `deploy-infra` | `infra/**` | `cloudbuild/terraform.yaml` | + +## Conventions + +- **Services communicate via HTTP contracts only.** No shared code between services. +- **Secrets** are managed via GCP Secret Manager. Local dev uses `.env` files. Agents have **read-only** access to secrets — ask the project owner to create new secrets. +- **Infrastructure** is managed with Terraform. Changes to `infra/` auto-deploy via Cloud Build. +- **Docker** is required for all services. Containers expose port 80 internally. +- **API endpoints** are the contract between frontend and backend. +{% if manifest.linear.enabled %} + +## Task Workflow (Linear) + +**All work must start with a Linear issue.** Do not make code changes without an associated task. + +### Before starting work: +1. Check the [{{ manifest.linear.team_key }} board](https://linear.app/{{ manifest.linear.workspace }}/team/{{ manifest.linear.team_key }}/active) for assigned tasks +2. If no task exists for the work you're about to do, **create one first** with: + - A clear, specific title (e.g., "Add pagination to /models endpoint" not "Fix API") + - A description with: Objective, Steps, Acceptance Criteria + - Appropriate priority (1=Urgent, 2=High, 3=Medium, 4=Low) + +### Working on a task: +1. Move the issue to **In Progress** +2. Create a branch named `{{ manifest.linear.team_key | lower }}-{issue-number}/{short-description}` (e.g., `ai4-42/add-pagination`) +3. Make your changes on the branch +4. Create a PR with the Linear issue ID in the title (e.g., "AI4-42: Add pagination to /models") +5. Linear auto-links the PR when the issue ID appears in the branch name or PR title + +### Completing a task: +1. Ensure the PR is merged to `main` +2. Cloud Build will auto-deploy the changes +3. Move the issue to **Done** + +### Issue descriptions must include: +- **Objective:** What and why (1-2 sentences) +- **Steps:** Numbered list of concrete actions +- **Branch:** Suggested branch name +- **Acceptance Criteria:** Bullet list of verifiable outcomes +- **Files:** List of files likely to change (if known) +{% endif %} + +## Development + +1. Install dependencies: `platform-cli install` +2. Start services: `docker compose up` +3. API: http://localhost:{{ manifest.services | selectattr('type', 'equalto', 'api') | map(attribute='port') | first | default(3006) }} +4. Web: http://localhost:{{ manifest.services | selectattr('type', 'equalto', 'webapp') | map(attribute='port') | first | default(3005) }} diff --git a/repoScaffold/templates/project/README.library-node.md.j2 b/repoScaffold/templates/project/README.library-node.md.j2 new file mode 100644 index 0000000..a8f1f93 --- /dev/null +++ b/repoScaffold/templates/project/README.library-node.md.j2 @@ -0,0 +1,35 @@ +# {{ manifest.project.name }} + +A publishable Node/npm package. +{% if manifest.linear.enabled %} + +## Want something built or fixed? + +**[Add a task on the {{ manifest.linear.team_key }} board →](https://linear.app/{{ manifest.linear.workspace }}/team/{{ manifest.linear.team_key }}/active)** + +Describe what you want — a new prop, an API improvement, a bug you hit, types, tests — and an agent will pick it up. Be specific: include how you use the library, what you saw, and what you expected. The owner reviews new tasks and promotes approved ones to **Todo** for the agent to work. +{% endif %} + +## Develop + +The package lives in `{{ manifest.library.package_dir }}/`. + +```bash +cd {{ manifest.library.package_dir }} +{{ manifest.library.build_cmd }} # build the package +{{ manifest.library.test_cmd }} # run tests +``` + +Releases are cut by the maintainer with `{{ manifest.library.publish_cmd }}` — agents never publish. + +## Dashboards + +| Resource | Link | +|----------|------| +{% if manifest.linear.enabled %} +| Project Board | [{{ manifest.linear.team_key }} — Linear](https://linear.app/{{ manifest.linear.workspace }}/team/{{ manifest.linear.team_key }}/active) | +{% endif %} + +## Project Config + +See `project.manifest.yaml` for the full project configuration. diff --git a/repoScaffold/templates/project/README.md.j2 b/repoScaffold/templates/project/README.md.j2 new file mode 100644 index 0000000..9e2d121 --- /dev/null +++ b/repoScaffold/templates/project/README.md.j2 @@ -0,0 +1,74 @@ +# {{ manifest.project.name }} + +{% if manifest.linear.enabled %} +## Want something built or fixed? + +**[Add a task on the {{ manifest.linear.team_key }} board →](https://linear.app/{{ manifest.linear.workspace }}/team/{{ manifest.linear.team_key }}/active)** + +Describe what you want — a new feature, a bug you hit, a UI change — and an agent will pick it up. Be specific: include what you saw, what you expected, and any relevant URLs or screenshots. The more detail, the faster it ships. + +{% endif %} +## Live Services + +| Service | URL / Schedule | +|---------|----------------| +{% for svc in manifest.services %} +{% if svc.enabled and svc.type == 'api' %} +| **API** | `https://{{ svc.name }}-PROJECTNUM.{{ manifest.project.cloud.region }}.run.app` | +{% endif %} +{% if svc.enabled and svc.type == 'webapp' %} +| **Web App** | `https://{{ svc.name }}-PROJECTNUM.{{ manifest.project.cloud.region }}.run.app` | +{% endif %} +{% if svc.enabled and svc.type == 'worker' %} +| **{{ svc.name | capitalize }}** (Cloud Run Job) | Scheduled: `{{ svc.schedule }}` | +{% endif %} +{% endfor %} + +## Quick Validation + +```bash +curl -sf https://api-PROJECTNUM.{{ manifest.project.cloud.region }}.run.app/health +curl -sf -o /dev/null -w "HTTP %{http_code}\n" https://app-PROJECTNUM.{{ manifest.project.cloud.region }}.run.app/ +``` + +## Dashboards + +| Resource | Link | +|----------|------| +{% if manifest.linear.enabled %} +| Project Board | [{{ manifest.linear.team_key }} — Linear](https://linear.app/{{ manifest.linear.workspace }}/team/{{ manifest.linear.team_key }}/active) | +{% endif %} +| Cloud Run | [GCP Console](https://console.cloud.google.com/run?project={{ manifest.project.cloud.project_id }}) | +| Cloud Build | [GCP Console](https://console.cloud.google.com/cloud-build/builds?project={{ manifest.project.cloud.project_id }}) | +| Secrets | [GCP Console](https://console.cloud.google.com/security/secret-manager?project={{ manifest.project.cloud.project_id }}) | +{% if manifest.database.type == 'mongodb' %} +| MongoDB Atlas | [Atlas Console — {{ manifest.database.atlas_cluster }}](https://cloud.mongodb.com/v2#/clusters) (db: `{{ manifest.database.db_name or manifest.project.name }}`) | +{% endif %} + +## Local Development + +```bash +docker compose up --build + +# API: http://localhost:{{ manifest.services | selectattr('type', 'equalto', 'api') | map(attribute='port') | first | default(3006) }} +# Web App: http://localhost:{{ manifest.services | selectattr('type', 'equalto', 'webapp') | map(attribute='port') | first | default(3005) }} +``` + +## How It Works + +Pushing to `main` triggers **Cloud Build** pipelines that build, push, and deploy automatically: + +| Trigger | Watches | Deploys | +|---------|---------|---------| +{% for svc in manifest.services %} +{% if svc.enabled %} +| `build-{{ svc.name }}` | `services/{{ 'api' if svc.type == 'api' else ('web' if svc.type == 'webapp' else 'worker') }}/**` | Cloud Run `{{ svc.name }}` | +{% endif %} +{% endfor %} +| `deploy-infra` | `infra/**` | Terraform apply | + +Infrastructure changes go through the same flow — edit Terraform, push, pipeline applies. + +## Project Config + +See `project.manifest.yaml` for the full project configuration. diff --git a/repoScaffold/templates/project/docker-compose.yml.j2 b/repoScaffold/templates/project/docker-compose.yml.j2 new file mode 100644 index 0000000..ed52cfa --- /dev/null +++ b/repoScaffold/templates/project/docker-compose.yml.j2 @@ -0,0 +1,36 @@ +services: +{% for svc in services %} +{% if svc.enabled %} +{% if svc.type == "api" %} + {{ svc.name }}: + build: + context: ./services/api + dockerfile: Dockerfile + ports: + - "{{ svc.port }}:80" + env_file: + - ./services/api/.env + volumes: + - ./services/api/src:/app/src + restart: unless-stopped +{% elif svc.type == "webapp" %} + {{ svc.name }}: + build: + context: ./services/web + dockerfile: Dockerfile + ports: + - "{{ svc.port }}:80" + depends_on: + - {{ manifest.services | selectattr('type', 'equalto', 'api') | map(attribute='name') | first | default('api') }} + restart: unless-stopped +{% elif svc.type == "worker" %} + {{ svc.name }}: + build: + context: ./services/worker + dockerfile: Dockerfile + env_file: + - ./services/worker/.env + restart: unless-stopped +{% endif %} +{% endif %} +{% endfor %} diff --git a/repoScaffold/templates/project/loop.sh.j2 b/repoScaffold/templates/project/loop.sh.j2 new file mode 100644 index 0000000..83415c7 --- /dev/null +++ b/repoScaffold/templates/project/loop.sh.j2 @@ -0,0 +1,325 @@ +#!/usr/bin/env bash +# {{ manifest.project.name }} — Agent loop runner +# Runs a single named agent in a loop. Each agent has its own PROMPT.md +# and config.yaml under agents/<name>/. +# +# Usage: +# ./loop.sh developer # Run the developer agent +# ./loop.sh reviewer # Run the reviewer agent +# ./loop.sh ops # Run the ops monitor agent +# +# The setup-vm.sh script starts all agents in separate tmux windows. + +AGENT_NAME="${1:?Usage: ./agents/loop.sh <agent-name>}" +PROJECT_ID="{{ manifest.project.cloud.project_id }}" +AGENTS_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$(cd "$AGENTS_DIR/.." && pwd)" +AGENT_DIR="$AGENTS_DIR/$AGENT_NAME" +LOG_TAG="agent-${AGENT_NAME}-${PROJECT_ID}" +LOG_DIR="/var/log/agent/${PROJECT_ID}/${AGENT_NAME}" + +# The main clone ($MAIN_REPO) holds the agent scripts and is the hot-reload +# source. Each agent does its actual git/file work in a PRIVATE worktree so +# concurrent agents never share a working tree — sharing one tree let uncommitted +# changes from one agent's task bleed into another's PR (10-file "package-lock" +# PRs, etc.). The worktree is checked out DETACHED at origin/main so it never +# collides with the main clone's `main` checkout. +MAIN_REPO="$REPO_DIR" +WORKTREE="${AGENT_WORKTREE:-$HOME/{{ manifest.project.name | lower | replace(' ', '-') }}-wt-$AGENT_NAME}" + +if [ ! -f "$AGENT_DIR/PROMPT.md" ]; then + echo "ERROR: $AGENT_NAME/PROMPT.md not found in agents/" + echo "Available agents:" + ls -1 "$AGENTS_DIR/" 2>/dev/null | while read -r d; do + [ -f "$AGENTS_DIR/$d/PROMPT.md" ] && echo " - $d" + done + exit 1 +fi + +# Read agent config (with defaults) +if [ -f "$AGENT_DIR/config.yaml" ]; then + SLEEP_SECONDS=$(grep -oP 'sleep_seconds:\s*\K\d+' "$AGENT_DIR/config.yaml" 2>/dev/null || echo 600) + MAX_TURNS=$(grep -oP 'max_turns:\s*\K\d+' "$AGENT_DIR/config.yaml" 2>/dev/null || echo 1000) +else + SLEEP_SECONDS=600 + MAX_TURNS=1000 +fi + +# Allow env overrides +SLEEP_SECONDS="${AGENT_SLEEP_SECONDS:-$SLEEP_SECONDS}" +MAX_TURNS="${AGENT_MAX_TURNS:-$MAX_TURNS}" + +# Board backpressure caps. Env-overridable. +# The review-queue signal is the number of Linear issues currently In Review — +# NOT the raw GitHub open-PR count. PRs the owner has demoted to Backlog stay +# open on GitHub but must NOT count as review load, or the board looks saturated +# while there is real Todo work waiting. +# Partial (developer agents): if In Review exceeds MAX_IN_REVIEW, developers stop +# starting new tasks (a full review queue just yields merge conflicts); if the +# Backlog exceeds MAX_BACKLOG, they stop creating tasks. +# Full saturation (ALL agents): when BOTH caps are exceeded there is nothing new +# to add and the human must clear the board — so every agent BACKS OFF for +# BACKOFF_SECONDS (a long sleep, no Claude call) and auto-resumes when cleared, +# instead of polling frequently. No restart needed. +MAX_IN_REVIEW="${AGENT_MAX_IN_REVIEW:-${AGENT_MAX_OPEN_PRS:-10}}" +MAX_BACKLOG="${AGENT_MAX_BACKLOG:-50}" +BACKOFF_SECONDS="${AGENT_BACKOFF_SECONDS:-3600}" +TEAM_KEY="${AGENT_LINEAR_TEAM_KEY:-{{ manifest.linear.team_key }}}" +REVIEW_STATE="${AGENT_LINEAR_REVIEW_STATE:-In Review}" +# One designated agent runs the deterministic board reconciliation each cycle +# (no Claude) — syncs Linear state to the real PR state (see reconcile.py). +RECONCILE_AGENT="${AGENT_RECONCILE_AGENT:-developer-1}" + +# ── Logging ───────────────────────────────────────────────────── + +log() { + local msg="$1" + echo "[$AGENT_NAME] $msg" + logger -t "$LOG_TAG" "$msg" 2>/dev/null || true +} + +# ── Board backpressure (cheap pre-Claude checks — no model tokens) ── + +# Number of issues currently In Review on the team — the review queue awaiting +# the owner. Queried from Linear (single request), NOT from GitHub: PRs the owner +# demoted to Backlog stay open on GitHub but are no longer In Review, so they +# correctly drop out of the backpressure signal. +count_in_review() { + local key; key="$(cat ~/.linear/api_key 2>/dev/null)" + [ -z "$key" ] && { echo 0; return; } + curl -s -H "Authorization: $key" -H "Content-Type: application/json" \ + -d "{\"query\":\"{ teams(filter:{key:{eq:\\\"$TEAM_KEY\\\"}}){ nodes{ issues(first:250, filter:{state:{name:{eq:\\\"$REVIEW_STATE\\\"}}}){ nodes{ id } } } } }\"}" \ + https://api.linear.app/graphql \ + | python3 -c 'import sys,json; d=json.load(sys.stdin)["data"]["teams"]["nodes"]; print(len(d[0]["issues"]["nodes"]) if d else 0)' 2>/dev/null || echo 0 +} + +# Number of Backlog issues on the team (queried by team key, single request). +count_backlog() { + local key; key="$(cat ~/.linear/api_key 2>/dev/null)" + [ -z "$key" ] && { echo 0; return; } + curl -s -H "Authorization: $key" -H "Content-Type: application/json" \ + -d "{\"query\":\"{ teams(filter:{key:{eq:\\\"$TEAM_KEY\\\"}}){ nodes{ issues(first:250, filter:{state:{type:{eq:\\\"backlog\\\"}}}){ nodes{ id } } } } }\"}" \ + https://api.linear.app/graphql \ + | python3 -c 'import sys,json; d=json.load(sys.stdin)["data"]["teams"]["nodes"]; print(len(d[0]["issues"]["nodes"]) if d else 0)' 2>/dev/null || echo 0 +} + +# Emit a saturation alert: a structured Cloud Logging marker (for a log-based +# email alert) plus a best-effort local email if a mailer is configured. +notify_saturation() { + local in_review="$1" backlog="$2" + logger -t "agent-alert-${PROJECT_ID}" \ + "AGENT_SATURATION_STOP agent=${AGENT_NAME} project=${PROJECT_ID} in_review=${in_review} max_in_review=${MAX_IN_REVIEW} backlog=${backlog} max_backlog=${MAX_BACKLOG}" 2>/dev/null || true + log "ALERT: board saturated — ${in_review} in review (> ${MAX_IN_REVIEW}) and ${backlog} backlog (> ${MAX_BACKLOG}). ${AGENT_NAME} backing off ${BACKOFF_SECONDS}s until cleared." + if [ -n "${AGENT_ALERT_EMAIL:-}" ] && command -v mail >/dev/null 2>&1; then + printf 'Agent %s (project %s) is backing off: the board is saturated.\n\nIn Review: %s (limit %s)\nBacklog: %s (limit %s)\n\nMerge the review queue or clear the backlog. Agents re-check every %ss and resume automatically — no restart needed.\n' \ + "$AGENT_NAME" "$PROJECT_ID" "$in_review" "$MAX_IN_REVIEW" "$backlog" "$MAX_BACKLOG" "$BACKOFF_SECONDS" \ + | mail -s "[agent-alert] ${PROJECT_ID}/${AGENT_NAME} backing off — board saturated" "$AGENT_ALERT_EMAIL" 2>/dev/null || true + fi +} + +# ── One-time setup ────────────────────────────────────────────── + +setup() { + log "=== Agent setup: $AGENT_NAME ===" + + sudo mkdir -p "$LOG_DIR" && sudo chown "$(whoami)" "$LOG_DIR" 2>/dev/null || true + + # Git auth + mkdir -p ~/.ssh + if [ ! -f ~/.ssh/deploy_key ] || [ ! -s ~/.ssh/deploy_key ]; then + log "Pulling deploy key from Secret Manager..." + gcloud secrets versions access latest \ + --secret=github-deploy-key \ + --project="$PROJECT_ID" > ~/.ssh/deploy_key 2>/dev/null || true + chmod 600 ~/.ssh/deploy_key 2>/dev/null || true + fi + export GIT_SSH_COMMAND="ssh -i ~/.ssh/deploy_key -o StrictHostKeyChecking=no" + + # Linear API key + mkdir -p ~/.linear + if [ ! -f ~/.linear/api_key ] || [ ! -s ~/.linear/api_key ]; then + log "Pulling Linear API key from Secret Manager..." + gcloud secrets versions access latest \ + --secret=linear-api-key \ + --project="$PROJECT_ID" > ~/.linear/api_key 2>/dev/null || \ + log "Warning: linear-api-key not found in Secret Manager." + fi + + # Claude Code auth — long-lived subscription OAuth token (from `claude setup-token`). + # Draws usage from the Claude subscription (not metered API billing) and is + # valid ~1 year, avoiding the interactive-login token expiry that takes the + # fleet down. Fetched via gcloud (never on a command line) so it's not exposed + # in `ps`. CLAUDE_CODE_OAUTH_TOKEN takes precedence over ~/.claude creds. + if [ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]; then + log "Pulling Claude Code OAuth token from Secret Manager..." + CLAUDE_CODE_OAUTH_TOKEN="$(gcloud secrets versions access latest \ + --secret=claude-code-oauth-token \ + --project="$PROJECT_ID" 2>/dev/null || true)" + export CLAUDE_CODE_OAUTH_TOKEN + fi + if [ -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]; then + log "ERROR: claude-code-oauth-token not found — Claude cannot authenticate." + fi + + cd "$REPO_DIR" + log "Working in: $REPO_DIR" + log "Agent: $AGENT_NAME" + log "Prompt: agents/$AGENT_NAME/PROMPT.md" + log "Sleep: ${SLEEP_SECONDS}s | Max turns: $MAX_TURNS" + log "Logs: tag=$LOG_TAG, local=$LOG_DIR/" +} + +# ── Disk cleanup ──────────────────────────────────────────────── + +cleanup_disk() { + ls -t "$LOG_DIR"/iteration-*.log 2>/dev/null | tail -n +6 | xargs rm -f 2>/dev/null || true + rm -rf /tmp/npm-* /tmp/pip-* 2>/dev/null || true +} + +# ── Per-agent worktree isolation ──────────────────────────────── +# Ensure this agent has its own private worktree, scrubbed to a pristine +# origin/main. Because the tree is private, the hard reset + clean are safe: +# no concurrent agent shares it, so nothing bleeds across tasks/PRs. +prepare_worktree() { + cd "$MAIN_REPO" || return 0 + git fetch origin main --quiet 2>/dev/null || true + + # Create the worktree once, detached at origin/main (detached avoids the + # "branch 'main' is already checked out" clash with the main clone). + if ! git worktree list --porcelain 2>/dev/null | grep -qx "worktree $WORKTREE"; then + rm -rf "$WORKTREE" 2>/dev/null || true + git worktree prune 2>/dev/null || true + git worktree add --detach "$WORKTREE" origin/main 2>/dev/null \ + || git worktree add --detach "$WORKTREE" main 2>/dev/null \ + || { log "WARN: could not create worktree $WORKTREE — falling back to main clone"; WORKTREE="$MAIN_REPO"; return 0; } + log "Created private worktree: $WORKTREE" + fi + + cd "$WORKTREE" || { WORKTREE="$MAIN_REPO"; return 0; } + # Scrub to a pristine, DETACHED origin/main: -f discards tracked edits and + # moves off any stale feature branch left by the previous iteration (so old + # local branches don't pile up); `git clean -fd` (no -x) removes untracked + # files but keeps ignored ones (node_modules, build caches). + git checkout -f --detach origin/main 2>/dev/null \ + || git checkout -f --detach main 2>/dev/null \ + || git reset --hard origin/main 2>/dev/null || true + git clean -fd 2>/dev/null || true + + # Seed installed deps from the main clone so each worktree doesn't reinstall. + # Use a hardlink copy (cp -al): near-instant, minimal disk, and a REAL dir + # (not a symlink) so the `node_modules/` gitignore rule actually matches it and + # `git clean -fd` preserves it. npm unlinks+rewrites files, so hardlinks don't + # corrupt the shared source. Only seeds once per worktree (skips if present). + for svc in {% for s in manifest.services %}services/{{ s.name }} {% endfor %}; do + if [ -d "$MAIN_REPO/$svc/node_modules" ] && [ ! -d "$WORKTREE/$svc/node_modules" ] && [ -d "$WORKTREE/$svc" ]; then + cp -al "$MAIN_REPO/$svc/node_modules" "$WORKTREE/$svc/node_modules" 2>/dev/null \ + || cp -a "$MAIN_REPO/$svc/node_modules" "$WORKTREE/$svc/node_modules" 2>/dev/null || true + fi + done +} + +# ── Main loop ─────────────────────────────────────────────────── + +run_iteration() { + local iteration=$1 + local log_file="$LOG_DIR/iteration-${iteration}.log" + + log "=== Iteration $iteration — $(date) ===" + + cleanup_disk + + # Stagger multi-instance agents so they don't all hit Linear at the same moment + JITTER=$(( RANDOM % 30 )) + log "Jitter: sleeping ${JITTER}s before starting" + sleep "$JITTER" || true + + # Run the task in this agent's PRIVATE worktree (isolated from other agents). + cd "$WORKTREE" 2>/dev/null || cd "$MAIN_REPO" + + PROMPT="$(cat "$AGENT_DIR/PROMPT.md")" + # Inject agent identity (multi-instance coordination) and, for developers, the + # current board counts + caps (set by the main loop) so Claude honors them + # without re-querying. + HEADER="[agent-id: ${AGENT_NAME}]" + case "$AGENT_NAME" in + developer*) HEADER="${HEADER} +[board-status: in_review=${IN_REVIEW} max_in_review=${MAX_IN_REVIEW} backlog=${BACKLOG} max_backlog=${MAX_BACKLOG}]" ;; + esac + PROMPT="${HEADER} + +${PROMPT}" + + claude -p "$PROMPT" \ + --allowedTools "Read,Edit,Write,Bash,Glob,Grep" \ + --max-turns "$MAX_TURNS" \ + --dangerously-skip-permissions \ + 2>&1 | tee "$log_file" || true + + log "=== Iteration $iteration complete — sleeping ${SLEEP_SECONDS}s ===" + + if [ -s "$log_file" ]; then + tail -50 "$log_file" | while IFS= read -r line; do + logger -t "$LOG_TAG" "iter=$iteration $line" 2>/dev/null || true + done + fi +} + +# ── Entry point ───────────────────────────────────────────────── + +setup + +# Hash of the running script, to detect updates pulled from main. +SELF_PATH="$AGENTS_DIR/loop.sh" +SELF_HASH="$(sha1sum "$SELF_PATH" 2>/dev/null | awk '{print $1}')" + +was_saturated=false +iteration=1 +while true; do + # Refresh the MAIN CLONE (agent scripts + hot-reload source) on latest main. + # Force to main: the main clone is no longer used for task work, so discarding + # any stray local state here is safe and keeps hot-reload deterministic. + cd "$MAIN_REPO" + git checkout -f main 2>/dev/null || true + git pull --rebase 2>/dev/null || git pull 2>/dev/null || true + + # Hot-reload: if loop.sh itself changed on main, re-exec the new version now — + # a safe boundary (no task running). Deploy loop/prompt changes by pushing to + # main; we never kill a running agent to pick up updates. + NEW_HASH="$(sha1sum "$SELF_PATH" 2>/dev/null | awk '{print $1}')" + if [ -n "$NEW_HASH" ] && [ "$NEW_HASH" != "$SELF_HASH" ]; then + log "loop.sh updated on main — reloading (no task interrupted)." + exec bash "$SELF_PATH" "$AGENT_NAME" + fi + + # Deterministic board reconciliation (one designated agent; no Claude). Syncs + # Linear state to the real PR state — merged->Done + close duplicate PRs, + # closed-only->Todo, stranded open PR->In Review. Runs before the saturation + # check so relieving duplicates/merges can lift a saturated board. + if [ "$AGENT_NAME" = "$RECONCILE_AGENT" ] && [ -f "$AGENTS_DIR/reconcile.py" ]; then + ( cd "$MAIN_REPO" && python3 "$AGENTS_DIR/reconcile.py" "$TEAM_KEY" ) || log "reconcile failed (non-fatal)" + fi + + # Board backpressure — applies to ALL agents. Cheap counts, no Claude call. + IN_REVIEW="$(count_in_review)"; BACKLOG="$(count_backlog)" + log "Board status: in_review=${IN_REVIEW}/${MAX_IN_REVIEW}, backlog=${BACKLOG}/${MAX_BACKLOG}" + if [ "${IN_REVIEW:-0}" -gt "$MAX_IN_REVIEW" ] && [ "${BACKLOG:-0}" -gt "$MAX_BACKLOG" ]; then + # Saturated: nothing new can be added and the human must clear the board. + # Back off for a long interval instead of polling / calling Claude; auto- + # resume when cleared. Alert once per entry into the saturated state. + if ! $was_saturated; then notify_saturation "$IN_REVIEW" "$BACKLOG"; was_saturated=true; fi + log "Board saturated — backing off ${BACKOFF_SECONDS}s (no Claude call). Auto-resumes when cleared." + sleep "$BACKOFF_SECONDS" || true + continue + fi + if $was_saturated; then + log "Board no longer saturated — resuming normal cadence." + was_saturated=false + fi + + # Prepare this agent's isolated, pristine worktree, then run the task in it. + prepare_worktree + + run_iteration "$iteration" || log "Iteration $iteration failed — continuing" + iteration=$((iteration + 1)) + sleep "$SLEEP_SECONDS" || true +done diff --git a/repoScaffold/templates/project/pr-screenshot.sh.j2 b/repoScaffold/templates/project/pr-screenshot.sh.j2 new file mode 100644 index 0000000..8bf75d0 --- /dev/null +++ b/repoScaffold/templates/project/pr-screenshot.sh.j2 @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# pr-screenshot.sh — upload a screenshot so it renders INLINE in a PR description. +# +# Why this exists: if the repo is PRIVATE, images committed to the branch or +# referenced via raw.githubusercontent.com / blob?raw=true do NOT render inline in +# a PR/issue body — the viewer's browser can't authenticate to those URLs +# cross-domain, so they 404 (this is why hand-written `![](file.png)` embeds are +# "always broken"). The GitHub web upload endpoint (user-attachments) needs a +# browser session, not a token, so agents can't use it either. +# +# What DOES work (private OR public): an unlisted ("secret") gist. Its raw URL is +# URL-addressable without auth, so GitHub serves it directly in the rendered body +# and it displays inline. We create a fresh secret gist per screenshot (no +# shared-gist push conflicts between concurrent agents) and git-push the binary +# PNG into it (the gists REST API only accepts text, so binary must go over git). +# +# Usage: +# agents/pr-screenshot.sh <image.png> ["alt text"] +# Prints a markdown image tag on stdout — paste it into the PR body: +# ![alt text](https://gist.githubusercontent.com/<user>/<id>/raw/<sha>/<file>) +# +# Requires: git, curl, and gh authenticated with a token that has `gist` scope +# (in addition to `repo`). A GitHub App installation token CANNOT create gists — +# the VM must be `gh auth login`'d with a user PAT that includes `gist`. +set -euo pipefail + +IMG="${1:?usage: pr-screenshot.sh <image.png> [alt-text]}" +ALT="${2:-screenshot}" + +[ -f "$IMG" ] || { echo "pr-screenshot: no such file: $IMG" >&2; exit 1; } +case "$(file -b --mime-type "$IMG" 2>/dev/null || echo)" in + image/*) : ;; + *) echo "pr-screenshot: not an image: $IMG" >&2; exit 1 ;; +esac + +TOKEN="$(gh auth token)" +[ -n "$TOKEN" ] || { echo "pr-screenshot: gh not authenticated" >&2; exit 1; } +BASE="$(basename "$IMG")" + +WORK="$(mktemp -d)" +META="$(mktemp)" +cleanup() { rm -rf "$WORK" "$META"; } +trap cleanup EXIT + +# 1. Create an unlisted gist (placeholder text file — REST gists take text only). +printf '{"public":false,"description":"PR screenshot ({{ manifest.project.name }}) — safe to delete","files":{"README.md":{"content":"PR screenshot upload."}}}' > "$META" +GIST_ID="$(gh api gists --input "$META" --jq '.id')" +[ -n "$GIST_ID" ] || { echo "pr-screenshot: gist create failed" >&2; exit 1; } +GIST_URL="https://${TOKEN}@gist.github.com/${GIST_ID}.git" + +# 2. Push the binary image into the gist over git. +git clone -q "$GIST_URL" "$WORK" +cp "$IMG" "$WORK/$BASE" +git -C "$WORK" -c user.email=agent@{{ manifest.project.name | lower | replace(' ', '-') }} -c user.name=agent add -- "$BASE" +git -C "$WORK" -c user.email=agent@{{ manifest.project.name | lower | replace(' ', '-') }} -c user.name=agent commit -q -m "add $BASE" +git -C "$WORK" push -q "$GIST_URL" HEAD + +# 3. Resolve the versioned raw URL and verify it serves the image before printing. +RAW="$(gh api "gists/${GIST_ID}" --jq ".files[\"${BASE}\"].raw_url")" +CODE="$(curl -s -o /dev/null -w '%{http_code}' -L "$RAW")" +[ "$CODE" = "200" ] || { echo "pr-screenshot: raw url not serving (HTTP $CODE): $RAW" >&2; exit 1; } + +printf '![%s](%s)\n' "$ALT" "$RAW" diff --git a/repoScaffold/templates/project/reconcile.py.j2 b/repoScaffold/templates/project/reconcile.py.j2 new file mode 100644 index 0000000..dbb9c34 --- /dev/null +++ b/repoScaffold/templates/project/reconcile.py.j2 @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Deterministic board reconciliation — no Claude, cheap, idempotent. + +Authoritatively syncs each Linear issue's state to the ACTUAL state of its +GitHub PR(s), fixing the race where a lagging agent resurrects a Done task or +opens a duplicate PR. Run by a single designated agent every iteration +(see loop.sh) — safe to re-run; it only acts when the board disagrees with the +PRs. + +Rules (per issue that has any PR): + - A PR is MERGED -> issue should be Done; close any still-OPEN duplicate PRs. + - Else issue is In Progress (open or closed PR) -> In Review, for the owner to triage. +Owner-controlled states (Done / Backlog / Canceled / Todo) are never touched. + +Usage: reconcile.py <TEAM_KEY> (or set AGENT_LINEAR_TEAM_KEY) +""" +import json +import os +import re +import subprocess +import sys + +TEAM_KEY = (sys.argv[1] if len(sys.argv) > 1 else os.environ.get("AGENT_LINEAR_TEAM_KEY", "")).strip() +DRY_RUN = os.environ.get("RECONCILE_DRYRUN") == "1" # log intended changes, mutate nothing +if not TEAM_KEY: + print("[reconcile] no team key (pass as arg or set AGENT_LINEAR_TEAM_KEY) — skipping", flush=True) + sys.exit(0) +try: + LINEAR_KEY = open(os.path.expanduser("~/.linear/api_key")).read().strip() +except OSError: + print("[reconcile] no linear key — skipping", flush=True) + sys.exit(0) + + +def log(msg): + print(f"[reconcile] {msg}", flush=True) + + +def gql(query): + r = subprocess.run( + ["curl", "-s", "-H", f"Authorization: {LINEAR_KEY}", + "-H", "Content-Type: application/json", + "-d", json.dumps({"query": query}), "https://api.linear.app/graphql"], + capture_output=True, text=True, timeout=30) + return json.loads(r.stdout or "{}") + + +def gh_all_prs(limit=150): + r = subprocess.run( + ["gh", "pr", "list", "--state", "all", "--limit", str(limit), + "--json", "number,title,headRefName,state"], + capture_output=True, text=True, timeout=60) + return json.loads(r.stdout or "[]") + + +def issue_number(pr): + m = re.search(rf"{re.escape(TEAM_KEY)}-(\d+)", f"{pr['title']} {pr['headRefName']}", re.IGNORECASE) + return int(m.group(1)) if m else None + + +def main(): + data = gql('{ teams(filter:{key:{eq:"%s"}}){ nodes{ id states{ nodes{ id name } } } } }' % TEAM_KEY) + try: + team = data["data"]["teams"]["nodes"][0] + except (KeyError, IndexError, TypeError): + log("could not resolve team — skipping") + return + team_id = team["id"] + states = {s["name"]: s["id"] for s in team["states"]["nodes"]} + + # Bucket PRs per issue by their true state (OPEN / CLOSED / MERGED). + by_issue = {} + for pr in gh_all_prs(): + n = issue_number(pr) + if n is None: + continue + b = by_issue.setdefault(n, {"OPEN": [], "CLOSED": [], "MERGED": []}) + if pr["state"] in b: + b[pr["state"]].append(pr["number"]) + if not by_issue: + return + + # Current Linear state for exactly those issues. + nums = ",".join(str(n) for n in by_issue) + res = gql('{ team(id:"%s"){ issues(first:250, filter:{number:{in:[%s]}}){ nodes{ id number state{name} } } } }' + % (team_id, nums)) + try: + issues = res["data"]["team"]["issues"]["nodes"] + except (KeyError, TypeError): + log("could not fetch issue states — skipping") + return + cur = {i["number"]: (i["id"], i["state"]["name"]) for i in issues} + + def move(issue_id, num, to_name): + if to_name not in states: + return + if DRY_RUN: + log(f"[dry-run] {TEAM_KEY}-{num}: would -> {to_name}") + return + gql('mutation { issueUpdate(id:"%s", input:{stateId:"%s"}){ success } }' % (issue_id, states[to_name])) + log(f"{TEAM_KEY}-{num}: -> {to_name}") + + def close_pr(num, reason): + if DRY_RUN: + log(f"[dry-run] would close PR #{num} ({reason})") + return + subprocess.run(["gh", "pr", "close", str(num), "--comment", reason], + capture_output=True, text=True, timeout=30) + log(f"closed PR #{num} ({reason})") + + changed = 0 + for n, prs in by_issue.items(): + if n not in cur: + continue + issue_id, state = cur[n] + if prs["MERGED"]: + # Work is in main -> Done (never override owner-controlled states). + if state in ("Todo", "In Progress", "In Review"): + move(issue_id, n, "Done"); changed += 1 + for pr in prs["OPEN"]: + close_pr(pr, f"Duplicate — {TEAM_KEY}-{n} was already completed by a merged PR. Closing this duplicate."); changed += 1 + elif state == "In Progress": + # Issue is stuck In Progress but its PR is open (stranded) or was + # closed by the owner. Either way, surface it to In Review so the + # owner can triage (merge/backlog/cancel). A closed PR is an owner + # signal, NOT "redo it": never auto-Todo, and never touch Done (a + # Done task whose only PR is closed is owner-controlled — leave it). + move(issue_id, n, "In Review"); changed += 1 + + log(f"done ({changed} correction(s))") + + +if __name__ == "__main__": + try: + main() + except Exception as e: # never let reconciliation crash the agent loop + log(f"error (non-fatal): {e}") diff --git a/repoScaffold/templates/project/setup-vm.sh.j2 b/repoScaffold/templates/project/setup-vm.sh.j2 new file mode 100644 index 0000000..c35a444 --- /dev/null +++ b/repoScaffold/templates/project/setup-vm.sh.j2 @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# {{ manifest.project.name }} — Agent VM setup script +# Installs all dependencies and starts all agents. Run once on the agent VM. +# +# ── Provision the VM first (one-time) ───────────────────────────────── +# The fleet runs several headless Claude Code agents concurrently, which is +# memory-hungry. Size the VM accordingly — small shared-core types (e.g. +# e2-medium / 2 vCPU / 4 GB) THRASH and stall under load. Recommended default +# for ~6 agents: e2-standard-8 (8 vCPU / 32 GB) with a 50 GB disk. Scale the +# machine type with the number of agents (budget ~2 vCPU + ~4 GB per agent). +# +# gcloud compute instances create {{ manifest.project.name | lower | replace(' ', '-') }}-agents \ +# --zone={{ manifest.project.cloud.region }}-a \ +# --machine-type=e2-standard-8 \ +# --boot-disk-size=50GB --boot-disk-type=pd-balanced \ +# --image-family=debian-12 --image-project=debian-cloud +# +# ── Then provision + start the agents on it ─────────────────────────── +# gcloud compute ssh {{ manifest.project.name | lower | replace(' ', '-') }}-agents \ +# --zone={{ manifest.project.cloud.region }}-a --command="bash -s" < agents/setup-vm.sh +# +# Each agent gets its own tmux window within a single session. +# Add a new agent by creating agents/<name>/PROMPT.md in the repo. +# +set -euo pipefail + +PROJECT_ID="{{ manifest.project.cloud.project_id }}" +{% set gh_owner_placeholder = "GITHUB_OWNER" %} +REPO_URL="git@github.com:{{ gh_owner_placeholder }}/{{ manifest.project.name | lower | replace(' ', '-') }}.git" +REPO_DIR="$HOME/{{ manifest.project.name | lower | replace(' ', '-') }}" +TMUX_SESSION="agents" + +echo "=== {{ manifest.project.name }} Agent VM Setup ===" + +# ── 1. System packages ───────────────────────────────────────── + +echo "[1/9] Installing system packages..." +sudo apt-get update -qq +sudo apt-get install -y -qq tmux git curl jq + +# ── 2. Node.js ────────────────────────────────────────────────── + +if ! command -v node &>/dev/null; then + echo "[2/9] Installing Node.js..." + curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -E bash - + sudo apt-get install -y -qq nodejs +else + echo "[2/9] Node.js already installed: $(node --version)" +fi + +# ── 3. Claude Code ────────────────────────────────────────────── + +mkdir -p ~/.npm-global +npm config set prefix ~/.npm-global +export PATH="$HOME/.npm-global/bin:$PATH" + +grep -q '.npm-global/bin' ~/.bashrc 2>/dev/null || { + echo 'export PATH="$HOME/.npm-global/bin:$PATH"' >> ~/.bashrc +} + +if ! command -v claude &>/dev/null; then + echo "[3/9] Installing Claude Code..." + npm install -g @anthropic-ai/claude-code +else + echo "[3/9] Claude Code already installed: $(claude --version)" +fi + +# ── 4. Playwright + Chromium ──────────────────────────────────── +# Agents capture UI screenshots for PRs; install the browser once, VM-wide. + +echo "[4/9] Installing Playwright..." +npx playwright install chromium --with-deps 2>/dev/null || true + +# ── 5. Deploy key + repo clone ────────────────────────────────── + +echo "[5/9] Setting up deploy key and cloning repo..." +mkdir -p ~/.ssh + +gcloud secrets versions access latest \ + --secret=github-deploy-key \ + --project="$PROJECT_ID" > ~/.ssh/deploy_key +chmod 600 ~/.ssh/deploy_key + +export GIT_SSH_COMMAND="ssh -i ~/.ssh/deploy_key -o StrictHostKeyChecking=no" + +grep -q 'GIT_SSH_COMMAND' ~/.bashrc 2>/dev/null || { + echo 'export GIT_SSH_COMMAND="ssh -i ~/.ssh/deploy_key -o StrictHostKeyChecking=no"' >> ~/.bashrc +} + +if [ -d "$REPO_DIR" ]; then + cd "$REPO_DIR" && git pull +else + git clone "$REPO_URL" "$REPO_DIR" + cd "$REPO_DIR" +fi + +# ── 6. Pre-install service dependencies ──────────────────────── +# Seeds node_modules in the main clone so each agent's private worktree can +# hardlink-copy them (loop.sh) instead of running a fresh install per iteration. + +echo "[6/9] Installing service dependencies..." +for svc_dir in {% for s in manifest.services %}services/{{ s.name }} {% endfor %}; do + if [ -f "$REPO_DIR/$svc_dir/package.json" ] && [ ! -d "$REPO_DIR/$svc_dir/node_modules" ]; then + echo " Installing $svc_dir..." + ( cd "$REPO_DIR/$svc_dir" && npm ci --no-audit --no-fund --legacy-peer-deps ) || true + else + echo " $svc_dir: node_modules present or no package.json" + fi +done + +# ── 7. Linear API key ────────────────────────────────────────── + +echo "[7/9] Pulling Linear API key..." +mkdir -p ~/.linear +gcloud secrets versions access latest \ + --secret=linear-api-key \ + --project="$PROJECT_ID" > ~/.linear/api_key 2>/dev/null || \ + echo "Warning: linear-api-key not found. Linear integration disabled." +chmod 600 ~/.linear/api_key 2>/dev/null || true + +# ── 8. Claude Code OAuth token (Claude Code auth) ────────────── + +echo "[8/9] Verifying Claude Code OAuth token..." +# Agents authenticate Claude via a long-lived subscription OAuth token from +# Secret Manager (loop.sh fetches it each iteration; from `claude setup-token`). +# It draws on the Claude subscription (not metered API billing) and is valid +# ~1 year. A missing token means every agent fails to authenticate, so surface +# it clearly here at provision time. +if gcloud secrets describe claude-code-oauth-token --project="$PROJECT_ID" &>/dev/null; then + echo " claude-code-oauth-token present — agents will authenticate via subscription OAuth." +else + echo " WARNING: claude-code-oauth-token not found in Secret Manager." + echo " Agents cannot authenticate Claude until it exists. Create it with:" + echo " printf %s \"\$(claude setup-token)\" | gcloud secrets create claude-code-oauth-token \\" + echo " --data-file=- --replication-policy=automatic --project=$PROJECT_ID" +fi + +# ── 9. Start all agents ──────────────────────────────────────── + +echo "[9/9] Discovering agents and starting tmux sessions..." + +tmux kill-session -t "$TMUX_SESSION" 2>/dev/null || true + +FIRST=true +for agent_dir in "$REPO_DIR"/agents/*/; do + agent_name=$(basename "$agent_dir") + if [ ! -f "$agent_dir/PROMPT.md" ]; then + echo " Skipping $agent_name (no PROMPT.md)" + continue + fi + + # Skip the base agent when numbered instances exist (e.g. skip "developer" + # if "developer-1" is present) so we don't run a duplicate of the fleet. + if ls "$REPO_DIR"/agents/"${agent_name}-"*/PROMPT.md &>/dev/null; then + echo " Skipping $agent_name (numbered instances found)" + continue + fi + + if [ "$FIRST" = true ]; then + tmux new-session -d -s "$TMUX_SESSION" -n "$agent_name" \ + "export PATH=$HOME/.npm-global/bin:\$PATH; \ + export GIT_SSH_COMMAND='ssh -i ~/.ssh/deploy_key -o StrictHostKeyChecking=no'; \ + cd $REPO_DIR && ./agents/loop.sh $agent_name" + FIRST=false + else + tmux new-window -t "$TMUX_SESSION" -n "$agent_name" \ + "export PATH=$HOME/.npm-global/bin:\$PATH; \ + export GIT_SSH_COMMAND='ssh -i ~/.ssh/deploy_key -o StrictHostKeyChecking=no'; \ + cd $REPO_DIR && ./agents/loop.sh $agent_name" + fi + + echo " Started: $agent_name" +done + +echo "" +echo "=== Setup complete ===" +echo "Agents running in tmux session '$TMUX_SESSION'" +echo "" +echo " Attach: tmux attach -t $TMUX_SESSION" +echo " List windows: tmux list-windows -t $TMUX_SESSION" +echo " Switch agent: Ctrl+B, <window-number>" +echo " Logs: /var/log/agent/${PROJECT_ID}/<agent-name>/" +echo " Cloud Logs: https://console.cloud.google.com/logs/query;query=logName%3D%22projects%2F${PROJECT_ID}%2Flogs%2Fagent-%22?project=${PROJECT_ID}" diff --git a/repoScaffold/templates/services/api/.dockerignore.j2 b/repoScaffold/templates/services/api/.dockerignore.j2 new file mode 100644 index 0000000..047b8fb --- /dev/null +++ b/repoScaffold/templates/services/api/.dockerignore.j2 @@ -0,0 +1,8 @@ +node_modules +.env +.env.* +.git +.gitignore +.vscode +.DS_Store +npm-debug.log* diff --git a/repoScaffold/templates/services/api/.eslintrc.json.j2 b/repoScaffold/templates/services/api/.eslintrc.json.j2 new file mode 100644 index 0000000..af7b7a7 --- /dev/null +++ b/repoScaffold/templates/services/api/.eslintrc.json.j2 @@ -0,0 +1,14 @@ +{ + "env": { + "node": true, + "es2021": true + }, + "extends": "eslint:recommended", + "parserOptions": { + "ecmaVersion": "latest" + }, + "rules": { + "no-unused-vars": ["warn", { "argsIgnorePattern": "^_" }], + "no-console": "off" + } +} diff --git a/repoScaffold/templates/services/api/Dockerfile.j2 b/repoScaffold/templates/services/api/Dockerfile.j2 new file mode 100644 index 0000000..cb0fa67 --- /dev/null +++ b/repoScaffold/templates/services/api/Dockerfile.j2 @@ -0,0 +1,26 @@ +FROM node:20-alpine AS builder + +WORKDIR /app + +COPY package.json package-lock.json* ./ +RUN if [ -f package-lock.json ]; then npm ci --omit=dev; else npm install --omit=dev; fi + +FROM node:20-alpine + +WORKDIR /app + +RUN addgroup -g 1001 -S appgroup && \ + adduser -S appuser -u 1001 -G appgroup + +COPY --from=builder /app/node_modules ./node_modules +COPY package.json ./ +COPY src/ ./src/ + +USER appuser + +EXPOSE 80 + +ENV PORT=80 +ENV NODE_ENV=production + +CMD ["node", "src/index.js"] diff --git a/repoScaffold/templates/services/api/db.js.j2 b/repoScaffold/templates/services/api/db.js.j2 new file mode 100644 index 0000000..c4bb82f --- /dev/null +++ b/repoScaffold/templates/services/api/db.js.j2 @@ -0,0 +1,20 @@ +{% raw %}const mongoose = require("mongoose"); + +const MONGODB_URI = process.env.MONGODB_URI; + +async function connectDB() { + if (!MONGODB_URI) { + throw new Error("MONGODB_URI environment variable is not set"); + } + await mongoose.connect(MONGODB_URI); + console.log("Connected to MongoDB"); +} + +function dbState() { + // 0 disconnected, 1 connected, 2 connecting, 3 disconnecting + const map = { 0: "disconnected", 1: "connected", 2: "connecting", 3: "disconnecting" }; + return map[mongoose.connection.readyState] || "unknown"; +} + +module.exports = { connectDB, dbState }; +{% endraw %} diff --git a/repoScaffold/templates/services/api/index.js.j2 b/repoScaffold/templates/services/api/index.js.j2 new file mode 100644 index 0000000..f5a4adb --- /dev/null +++ b/repoScaffold/templates/services/api/index.js.j2 @@ -0,0 +1,47 @@ +require("dotenv").config(); + +const express = require("express"); +const cors = require("cors"); +const helmet = require("helmet"); +{% raw %} +const { connectDB, dbState } = require("./db"); +const { Item } = require("./models"); +{% endraw %} + +const app = express(); +const PORT = process.env.PORT || {{ service.port }}; + +app.use(helmet()); +app.use(cors()); +app.use(express.json()); + +app.get("/health", (_req, res) => { +{% raw %} + res.json({ status: "ok", service: "{% endraw %}{{ service.name }}{% raw %}", db: dbState() }); +{% endraw %} +}); + +app.get("/items", async (_req, res) => { +{% raw %} + if (dbState() !== "connected") { + return res.status(503).json({ error: "database unavailable", db: dbState() }); + } + try { + const items = await Item.find().lean(); + res.json(items); + } catch (err) { + console.error("GET /items error:", err.message); + res.status(500).json({ error: "Internal server error" }); + } +{% endraw %} +}); + +app.listen(PORT, () => { +{% raw %} + console.log(`API listening on port ${PORT}`); +}); + +connectDB().catch((err) => { + console.error("DB connect failed (API still running):", err.message); +}); +{% endraw %} diff --git a/repoScaffold/templates/services/api/models.js.j2 b/repoScaffold/templates/services/api/models.js.j2 new file mode 100644 index 0000000..7838128 --- /dev/null +++ b/repoScaffold/templates/services/api/models.js.j2 @@ -0,0 +1,14 @@ +{% raw %} +const mongoose = require("mongoose"); + +const itemSchema = new mongoose.Schema( + { + name: { type: String, required: true }, + }, + { timestamps: true } +); + +const Item = mongoose.model("Item", itemSchema); + +module.exports = { Item }; +{% endraw %} diff --git a/repoScaffold/templates/services/api/package.json.j2 b/repoScaffold/templates/services/api/package.json.j2 new file mode 100644 index 0000000..f727bf2 --- /dev/null +++ b/repoScaffold/templates/services/api/package.json.j2 @@ -0,0 +1,22 @@ +{ + "name": "{{ project.name | lower | replace(' ', '-') }}-api", + "version": "1.0.0", + "description": "{{ project.name }} API service", + "main": "src/index.js", + "scripts": { + "start": "node src/index.js", + "dev": "node --watch src/index.js", + "lint": "eslint src/", + "lint:fix": "eslint src/ --fix" + }, + "dependencies": { + "express": "^4.18.2", + "cors": "^2.8.5", + "dotenv": "^16.3.1", + "mongoose": "^8.0.0", + "helmet": "^7.1.0" + }, + "devDependencies": { + "eslint": "^8.56.0" + } +} diff --git a/repoScaffold/templates/services/web/.dockerignore.j2 b/repoScaffold/templates/services/web/.dockerignore.j2 new file mode 100644 index 0000000..e07f5a0 --- /dev/null +++ b/repoScaffold/templates/services/web/.dockerignore.j2 @@ -0,0 +1,9 @@ +node_modules +dist +.env +.env.* +.git +.gitignore +.vscode +.DS_Store +npm-debug.log* diff --git a/repoScaffold/templates/services/web/App.jsx.j2 b/repoScaffold/templates/services/web/App.jsx.j2 new file mode 100644 index 0000000..6727deb --- /dev/null +++ b/repoScaffold/templates/services/web/App.jsx.j2 @@ -0,0 +1,34 @@ +{% raw %}import { useEffect, useState } from "react"; +import { fetchItems } from "./api"; + +export default function App() { + const [items, setItems] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + fetchItems() + .then(setItems) + .catch((err) => setError(err.message)) + .finally(() => setLoading(false)); + }, []); + + if (loading) return <div>Loading...</div>; + if (error) return <div>Error: {error}</div>; + + return ( + <div style={{ padding: "2rem", fontFamily: "system-ui, sans-serif" }}> + <h1>Items</h1> + {items.length === 0 ? ( + <p>No items found.</p> + ) : ( + <ul> + {items.map((item) => ( + <li key={item._id || item.name}>{item.name}</li> + ))} + </ul> + )} + </div> + ); +} +{% endraw %} \ No newline at end of file diff --git a/repoScaffold/templates/services/web/Dockerfile.j2 b/repoScaffold/templates/services/web/Dockerfile.j2 new file mode 100644 index 0000000..ab00d2c --- /dev/null +++ b/repoScaffold/templates/services/web/Dockerfile.j2 @@ -0,0 +1,26 @@ +FROM node:20-alpine AS builder + +WORKDIR /app + +COPY package.json package-lock.json* ./ +RUN if [ -f package-lock.json ]; then npm ci; else npm install; fi + +COPY . . +RUN npm run build + +FROM nginx:alpine + +COPY --from=builder /app/dist /usr/share/nginx/html + +RUN printf 'server {\n\ + listen 80;\n\ + root /usr/share/nginx/html;\n\ + index index.html;\n\ + location / {\n\ + try_files $uri $uri/ /index.html;\n\ + }\n\ +}\n' > /etc/nginx/conf.d/default.conf + +EXPOSE 80 + +CMD ["nginx", "-g", "daemon off;"] diff --git a/repoScaffold/templates/services/web/api.js.j2 b/repoScaffold/templates/services/web/api.js.j2 new file mode 100644 index 0000000..ae1a2a8 --- /dev/null +++ b/repoScaffold/templates/services/web/api.js.j2 @@ -0,0 +1,10 @@ +{% raw %}const API_BASE = import.meta.env.VITE_API_URL || ""; + +export async function fetchItems() { + const res = await fetch(`${API_BASE}/items`); + if (!res.ok) { + throw new Error(`Failed to fetch items: ${res.status}`); + } + return res.json(); +} +{% endraw %} \ No newline at end of file diff --git a/repoScaffold/templates/services/web/index.html.j2 b/repoScaffold/templates/services/web/index.html.j2 new file mode 100644 index 0000000..cf9bbda --- /dev/null +++ b/repoScaffold/templates/services/web/index.html.j2 @@ -0,0 +1,12 @@ +<!doctype html> +<html lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <title>{{ project.name }} + + +
+ + + diff --git a/repoScaffold/templates/services/web/main.jsx.j2 b/repoScaffold/templates/services/web/main.jsx.j2 new file mode 100644 index 0000000..4aaf0e6 --- /dev/null +++ b/repoScaffold/templates/services/web/main.jsx.j2 @@ -0,0 +1,10 @@ +{% raw %}import { StrictMode } from "react"; +import { createRoot } from "react-dom/client"; +import App from "./App.jsx"; + +createRoot(document.getElementById("root")).render( + + + , +); +{% endraw %} \ No newline at end of file diff --git a/repoScaffold/templates/services/web/package.json.j2 b/repoScaffold/templates/services/web/package.json.j2 new file mode 100644 index 0000000..ca9cb83 --- /dev/null +++ b/repoScaffold/templates/services/web/package.json.j2 @@ -0,0 +1,19 @@ +{ + "name": "{{ project.name | lower | replace(' ', '-') }}-web", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@vitejs/plugin-react": "^4.3.1", + "vite": "^5.4.0" + } +} diff --git a/repoScaffold/templates/services/web/vite.config.js.j2 b/repoScaffold/templates/services/web/vite.config.js.j2 new file mode 100644 index 0000000..7330a57 --- /dev/null +++ b/repoScaffold/templates/services/web/vite.config.js.j2 @@ -0,0 +1,19 @@ +{% raw %}import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +export default defineConfig({ + plugins: [react()], + server: { + host: true, + port: {% endraw %}{{ web_port }}{% raw %}, + proxy: { + "/items": "http://localhost:{% endraw %}{{ api_port }}{% raw %}", + "/health": "http://localhost:{% endraw %}{{ api_port }}{% raw %}", + }, + }, + preview: { + host: true, + port: {% endraw %}{{ web_port }}{% raw %}, + }, +}); +{% endraw %} \ No newline at end of file diff --git a/repoScaffold/templates/services/worker/Dockerfile.j2 b/repoScaffold/templates/services/worker/Dockerfile.j2 new file mode 100644 index 0000000..b1706eb --- /dev/null +++ b/repoScaffold/templates/services/worker/Dockerfile.j2 @@ -0,0 +1,16 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN addgroup --gid 1001 appgroup && \ + adduser --uid 1001 --gid 1001 --disabled-password appuser + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +USER appuser + +# Runs as a Cloud Run Job (one-shot script). No HTTP server, no port. +CMD ["python", "main.py"] diff --git a/repoScaffold/templates/services/worker/main.py.j2 b/repoScaffold/templates/services/worker/main.py.j2 new file mode 100644 index 0000000..ab942de --- /dev/null +++ b/repoScaffold/templates/services/worker/main.py.j2 @@ -0,0 +1,29 @@ +"""{{ service.name }} worker — runs as a scheduled Cloud Run Job. + +Triggered by Cloud Scheduler on cron schedule: {{ service.schedule }} +""" +import logging +import os +import sys + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", +) +logger = logging.getLogger(__name__) + + +def main() -> int: + """Entry point for the worker. Runs to completion, then exits.""" + logger.info("Worker starting (task index=%s)", os.environ.get("CLOUD_RUN_TASK_INDEX", "0")) + + # TODO: Implement worker logic here. + # This is a no-op placeholder that just logs and exits. + logger.info("Hello from {{ service.name }}!") + + logger.info("Worker completed successfully.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/repoScaffold/templates/services/worker/requirements.txt.j2 b/repoScaffold/templates/services/worker/requirements.txt.j2 new file mode 100644 index 0000000..f9949ff --- /dev/null +++ b/repoScaffold/templates/services/worker/requirements.txt.j2 @@ -0,0 +1 @@ +google-cloud-logging>=3.8.0 diff --git a/repoScaffold/templates/vscode/launch.json.j2 b/repoScaffold/templates/vscode/launch.json.j2 new file mode 100644 index 0000000..a3a4a4d --- /dev/null +++ b/repoScaffold/templates/vscode/launch.json.j2 @@ -0,0 +1,33 @@ +{% raw %} +{ + "version": "0.2.0", + "configurations": [ + { + "name": "API: Node.js", + "type": "node", + "request": "launch", + "program": "${workspaceFolder}/services/api/src/index.js", + "envFile": "${workspaceFolder}/services/api/.env", + "restart": true, + "console": "integratedTerminal" + }, + { + "name": "Web: React Dev Server", + "type": "node", + "request": "launch", + "runtimeExecutable": "npm", + "runtimeArgs": ["start"], + "cwd": "${workspaceFolder}/services/web", + "console": "integratedTerminal" + }, + { + "name": "Docker Compose Up", + "type": "node", + "request": "launch", + "runtimeExecutable": "docker", + "runtimeArgs": ["compose", "up", "--build"], + "console": "integratedTerminal" + } + ] +} +{% endraw %} diff --git a/repoScaffold/templates/vscode/tasks.json.j2 b/repoScaffold/templates/vscode/tasks.json.j2 new file mode 100644 index 0000000..851b5e1 --- /dev/null +++ b/repoScaffold/templates/vscode/tasks.json.j2 @@ -0,0 +1,49 @@ +{% raw %} +{ + "version": "2.0.0", + "tasks": [ + { + "label": "API: Install", + "type": "shell", + "command": "npm install", + "options": { "cwd": "${workspaceFolder}/services/api" } + }, + { + "label": "API: Start", + "type": "shell", + "command": "npm start", + "options": { "cwd": "${workspaceFolder}/services/api" }, + "isBackground": true + }, + { + "label": "Web: Install", + "type": "shell", + "command": "npm install", + "options": { "cwd": "${workspaceFolder}/services/web" } + }, + { + "label": "Web: Start", + "type": "shell", + "command": "npm start", + "options": { "cwd": "${workspaceFolder}/services/web" }, + "isBackground": true + }, + { + "label": "Docker: Build All", + "type": "shell", + "command": "docker compose build" + }, + { + "label": "Docker: Up", + "type": "shell", + "command": "docker compose up" + }, + { + "label": "Terraform: Plan", + "type": "shell", + "command": "terraform plan", + "options": { "cwd": "${workspaceFolder}/infra/terraform/envs/dev" } + } + ] +} +{% endraw %}