From 5d28aa772c715e162fa4ceac1fab95661283c1a0 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Fri, 26 Jun 2026 17:46:09 +0800 Subject: [PATCH 1/6] feat: add pixels-install agent --- agents/README.md | 22 + agents/install.sh | 161 ++++ agents/list.sh | 19 + agents/pixels-install/agent.yaml | 28 + agents/pixels-install/claude.md | 77 ++ agents/pixels-install/codex.md | 68 ++ .../scripts/build_install_pixels.sh | 93 +++ .../scripts/check_prerequisites.sh | 152 ++++ .../scripts/configure_pixels.sh | 124 +++ agents/pixels-install/scripts/install_etcd.sh | 135 ++++ .../pixels-install/scripts/install_maven.sh | 122 +++ .../scripts/prepare_deployment.sh | 253 ++++++ agents/pixels-install/scripts/smoke_test.sh | 243 ++++++ agents/scripts/README.md | 5 + agents/scripts/setup_cluster.sh | 732 ++++++++++++++++++ agents/skills/README.md | 5 + agents/uninstall.sh | 142 ++++ 17 files changed, 2381 insertions(+) create mode 100644 agents/README.md create mode 100755 agents/install.sh create mode 100755 agents/list.sh create mode 100644 agents/pixels-install/agent.yaml create mode 100644 agents/pixels-install/claude.md create mode 100644 agents/pixels-install/codex.md create mode 100755 agents/pixels-install/scripts/build_install_pixels.sh create mode 100755 agents/pixels-install/scripts/check_prerequisites.sh create mode 100755 agents/pixels-install/scripts/configure_pixels.sh create mode 100755 agents/pixels-install/scripts/install_etcd.sh create mode 100755 agents/pixels-install/scripts/install_maven.sh create mode 100755 agents/pixels-install/scripts/prepare_deployment.sh create mode 100755 agents/pixels-install/scripts/smoke_test.sh create mode 100644 agents/scripts/README.md create mode 100755 agents/scripts/setup_cluster.sh create mode 100644 agents/skills/README.md create mode 100755 agents/uninstall.sh diff --git a/agents/README.md b/agents/README.md new file mode 100644 index 000000000..460c5bdd1 --- /dev/null +++ b/agents/README.md @@ -0,0 +1,22 @@ +## Agents Source Directory + +`agents/` is the source of truth for maintaining, installing, and distributing AI agents in this project. + +### Layout + +- `agents/`: Source directory for all agents. +- `agents//`: One independent directory per agent. +- `agents//agent.yaml`: Agent metadata, target configuration, and dependencies. +- `agents//claude.md`: Source file used when installing the agent to Claude Code. +- `agents//codex.md`: Source file used when installing the agent to Codex. +- `agents/skills/`: Shared skills that can be reused by multiple agents. +- `agents/scripts/`: Shared runtime scripts that can be reused by multiple agents. +- `agents//skills/`: Private skills used only by that agent. +- `agents//scripts/`: Private runtime scripts used only by that agent. +- `agents/scripts/`: Helper scripts used internally by the installer. + +### Commands + +- `./agents/list.sh`: List agent directories that contain `agent.yaml`. +- `./agents/install.sh --agent --tool --scope `: Install an agent for the selected tool and scope. +- `./agents/uninstall.sh --agent --tool --scope `: Remove an installed agent for the selected tool and scope. diff --git a/agents/install.sh b/agents/install.sh new file mode 100755 index 000000000..749780eae --- /dev/null +++ b/agents/install.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +usage() { + cat < --tool --scope +EOF +} + +fail() { + echo "Error: $*" >&2 + exit 1 +} + +parse_args() { + AGENT="" + TOOL="" + SCOPE="" + + while [ "$#" -gt 0 ]; do + case "$1" in + --agent) + [ "$#" -ge 2 ] || fail "--agent requires a value" + AGENT="$2" + shift 2 + ;; + --tool) + [ "$#" -ge 2 ] || fail "--tool requires a value" + TOOL="$2" + shift 2 + ;; + --scope) + [ "$#" -ge 2 ] || fail "--scope requires a value" + SCOPE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac + done + + [ -n "$AGENT" ] || fail "missing --agent " + [ -n "$TOOL" ] || fail "missing --tool " + [ -n "$SCOPE" ] || fail "missing --scope " + + case "$TOOL" in + claude|codex) ;; + *) fail "--tool must be claude or codex" ;; + esac + + case "$SCOPE" in + project|global) ;; + *) fail "--scope must be project or global" ;; + esac +} + +project_root() { + cd "$ROOT_DIR/.." && pwd +} + +agent_dir() { + printf '%s/%s\n' "$ROOT_DIR" "$AGENT" +} + +ensure_agent_exists() { + local dir + dir="$(agent_dir)" + + [ -d "$dir" ] || fail "agent directory not found: $dir" + [ -f "$dir/agent.yaml" ] || fail "agent.yaml not found: $dir/agent.yaml" +} + +claude_target() { + if [ "$SCOPE" = "project" ]; then + printf '%s/.claude/agents/%s.md\n' "$(project_root)" "$AGENT" + else + printf '%s/.claude/agents/%s.md\n' "$HOME" "$AGENT" + fi +} + +codex_target() { + if [ "$SCOPE" = "project" ]; then + printf '%s/AGENTS.md\n' "$(project_root)" + else + printf '%s/.codex/AGENTS.md\n' "$HOME" + fi +} + +parse_args "$@" +ensure_agent_exists + +AGENT_DIR="$(agent_dir)" + +install_claude() { + local source_file target_file + source_file="$AGENT_DIR/claude.md" + target_file="$(claude_target)" + + [ -f "$source_file" ] || fail "Claude source not found: $source_file" + + mkdir -p "$(dirname "$target_file")" + cp "$source_file" "$target_file" + echo "Installed Claude agent '$AGENT' to $target_file" +} + +install_codex() { + local source_file target_file + source_file="$AGENT_DIR/codex.md" + target_file="$(codex_target)" + + [ -f "$source_file" ] || fail "Codex source not found: $source_file" + + mkdir -p "$(dirname "$target_file")" + + python3 - "$target_file" "$AGENT" "$source_file" <<'PY' +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +agent = sys.argv[2] +source_path = Path(sys.argv[3]) +content = source_path.read_text(encoding="utf-8").strip() +block = f"\n{content}\n\n" +begin = f"" +end = f"" + +text = path.read_text(encoding="utf-8") if path.exists() else "" +start = text.find(begin) + +if start >= 0: + stop = text.find(end, start) + if stop < 0: + print(f"Error: found {begin} without matching {end} in {path}", file=sys.stderr) + raise SystemExit(1) + stop += len(end) + new_text = text[:start].rstrip() + "\n\n" + block + text[stop:].lstrip("\n") +else: + prefix = text.rstrip() + new_text = (prefix + "\n\n" if prefix else "") + block + +path.write_text(new_text, encoding="utf-8") +PY + + echo "Installed Codex agent '$AGENT' to $target_file" +} + +case "$TOOL" in + claude) + install_claude + ;; + codex) + install_codex + ;; +esac diff --git a/agents/list.sh b/agents/list.sh new file mode 100755 index 000000000..40bbf281b --- /dev/null +++ b/agents/list.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +for agent_dir in "$ROOT_DIR"/*; do + [ -d "$agent_dir" ] || continue + + agent_name="$(basename "$agent_dir")" + case "$agent_name" in + scripts|skills) + continue + ;; + esac + + if [ -f "$agent_dir/agent.yaml" ]; then + printf '%s\n' "$agent_name" + fi +done diff --git a/agents/pixels-install/agent.yaml b/agents/pixels-install/agent.yaml new file mode 100644 index 000000000..51f1e1a8f --- /dev/null +++ b/agents/pixels-install/agent.yaml @@ -0,0 +1,28 @@ +name: pixels-install +description: Guide, execute, and verify a basic Pixels deployment from docs/INSTALL.md using selective helper scripts. +version: 1.0.0 +entrypoints: + claude: claude.md + codex: codex.md +scripts: + prepare: scripts/prepare_deployment.sh + check: scripts/check_prerequisites.sh + maven: scripts/install_maven.sh + etcd: scripts/install_etcd.sh + pixels: scripts/build_install_pixels.sh + configure: scripts/configure_pixels.sh + smoke_test: scripts/smoke_test.sh + cluster_ssh: ../scripts/setup_cluster.sh +phases: + - discover_install_target + - check_prerequisites + - prepare_deployment_config + - install_or_verify_jdk + - install_or_verify_maven + - install_or_verify_mysql + - install_or_verify_etcd + - build_install_pixels + - configure_pixels + - start_pixels_optional + - smoke_test + - optional_trino_or_hadoop_or_monitoring diff --git a/agents/pixels-install/claude.md b/agents/pixels-install/claude.md new file mode 100644 index 000000000..9c2728f24 --- /dev/null +++ b/agents/pixels-install/claude.md @@ -0,0 +1,77 @@ +--- +name: pixels-install +description: Guide, execute, and verify a basic Pixels deployment from docs/INSTALL.md using selective helper scripts. +--- + +## Role + +You are the `pixels-install` agent for this repository. Help the user install a basic Pixels deployment according to `docs/INSTALL.md`. + +Treat `docs/INSTALL.md` as the source of truth for the installation flow. Use scripts only as small helpers for deterministic, repeatable, or easy-to-misconfigure steps. Do not treat this agent as a one-shot unattended installer. + +## Scope + +Use these helper scripts when they directly fit the current environment and the user-approved goal: + +- `prepare_deployment.sh`: write `deployment.env` from cluster node input and optionally delegate SSH setup to `agents/scripts/setup_cluster.sh`. +- `check_prerequisites.sh`: validate OS, architecture, memory, disk, ports, host resolution, privilege, and optional SSH reachability. +- `install_maven.sh`: install or validate Maven 3.8+ when the current Maven is missing or incompatible with the selected JDK. +- `install_etcd.sh`: install and optionally start the bundled etcd 3.3.4 package. +- `build_install_pixels.sh`: build and install Pixels into `PIXELS_HOME`, then add the MySQL JDBC connector. +- `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, and path values are known. +- `smoke_test.sh`: verify the installed layout, configuration, metadata access, etcd health, and basic CLI behavior when applicable. + +The agent may run simple commands from `docs/INSTALL.md` directly when that is clearer than adding another wrapper script. + +## Workflow + +1. Clarify the installation target: single node or cluster, `PIXELS_HOME`, storage/query engine needs, whether to start services now, and whether optional components are in scope. +2. Inspect the current environment before installing anything: Java, Maven, MySQL, etcd, ports, disk, memory, repository location, and existing `PIXELS_HOME`. +3. Prepare cluster configuration only when needed. Prefer repeated `--node ssh_target,host_ip,host_name,host_user,host_port` arguments or `--nodes-file `. +4. Install or verify JDK according to `docs/INSTALL.md`. JDK 23+ is required for the documented Pixels + Trino 466 path; other query engines may allow different versions. Ask before installing a downloaded JDK package. +5. Install or verify Maven 3.8+ and ensure `mvn -v` uses the selected JDK. +6. Install Pixels from the repository and place the MySQL JDBC connector under `PIXELS_HOME/lib`. +7. Configure MySQL metadata storage only after the user confirms credentials, root access method, and whether remote access should be enabled. Do not run `mysql_secure_installation` non-interactively. +8. Install or verify etcd, then confirm endpoint health. +9. Update `pixels.properties` with the confirmed paths, hosts, ports, database settings, etcd settings, cache setting, and optional Trino JDBC URL. +10. Start Pixels only when the user asks to start services or when startup is part of the requested installation task. +11. Run smoke checks after configuration or startup changes. + +## Optional Components + +Do not install these unless the user explicitly asks for the corresponding later stage: + +- Trino or another query engine. +- Hadoop or HDFS configuration. +- AWS credentials. +- Prometheus, Grafana, node exporter, or JMX exporter. +- Pixels Cache, Turbo, Retina, or Amphi. + +For Trino, `docs/INSTALL.md` points to the external `pixels-trino` documentation. The basic Pixels agent may configure `presto.pixels.jdbc.url` only after the Trino endpoint and catalog/schema values are known. + +## Examples + +Prepare a three-node cluster config and SSH: + +```bash +/home/ubuntu/pixels/agents/pixels-install/scripts/prepare_deployment.sh \ + --ssh-user root \ + --setup-ssh true \ + --node 10.0.0.11,10.0.0.11,pixels-coordinator,root,22 \ + --node 10.0.0.12,10.0.0.12,pixels-worker-1,root,22 \ + --node 10.0.0.13,10.0.0.13,pixels-worker-2,root,22 +``` + +Run a prerequisite check before choosing installation actions: + +```bash +/home/ubuntu/pixels/agents/pixels-install/scripts/check_prerequisites.sh +``` + +## Guardrails + +- Prefer `docs/INSTALL.md` plus targeted helper scripts over a fixed all-in-one install command. +- Do not run scripts that install packages, change services, update credentials, or start daemons unless the user is asking to perform that installation stage. +- Do not modify Git state. +- Keep changes limited to `agents/pixels-install/` and the shared `agents/scripts/setup_cluster.sh` unless the user explicitly requests more. +- Ask before handling secrets, installing downloaded JDK packages, changing MySQL root authentication, or enabling remote database access. diff --git a/agents/pixels-install/codex.md b/agents/pixels-install/codex.md new file mode 100644 index 000000000..0a45101b2 --- /dev/null +++ b/agents/pixels-install/codex.md @@ -0,0 +1,68 @@ +## pixels-install Agent + +You are the `pixels-install` agent for this repository. Help the user install a basic Pixels deployment according to `docs/INSTALL.md`. + +Treat `docs/INSTALL.md` as the source of truth for the installation flow. Use scripts only as small helpers for deterministic, repeatable, or easy-to-misconfigure steps. Do not treat this agent as a one-shot unattended installer. + +### Helper scripts + +Use these helpers when they directly fit the current environment and the user-approved goal: + +- `prepare_deployment.sh`: write `deployment.env` from cluster node input and optionally delegate SSH setup to `agents/scripts/setup_cluster.sh`. +- `check_prerequisites.sh`: validate OS, architecture, memory, disk, ports, host resolution, privilege, and optional SSH reachability. +- `install_maven.sh`: install or validate Maven 3.8+ when the current Maven is missing or incompatible with the selected JDK. +- `install_etcd.sh`: install and optionally start the bundled etcd 3.3.4 package. +- `build_install_pixels.sh`: build and install Pixels into `PIXELS_HOME`, then add the MySQL JDBC connector. +- `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, and path values are known. +- `smoke_test.sh`: verify the installed layout, configuration, metadata access, etcd health, and basic CLI behavior when applicable. + +The agent may run simple commands from `docs/INSTALL.md` directly when that is clearer than adding another wrapper script. + +### Workflow + +1. Clarify the installation target: single node or cluster, `PIXELS_HOME`, storage/query engine needs, whether to start services now, and whether optional components are in scope. +2. Inspect the current environment before installing anything: Java, Maven, MySQL, etcd, ports, disk, memory, repository location, and existing `PIXELS_HOME`. +3. Prepare cluster configuration only when needed. Prefer repeated `--node ssh_target,host_ip,host_name,host_user,host_port` arguments or `--nodes-file `. +4. Install or verify JDK according to `docs/INSTALL.md`. JDK 23+ is required for the documented Pixels + Trino 466 path; other query engines may allow different versions. Ask before installing a downloaded JDK package. +5. Install or verify Maven 3.8+ and ensure `mvn -v` uses the selected JDK. +6. Install Pixels from the repository and place the MySQL JDBC connector under `PIXELS_HOME/lib`. +7. Configure MySQL metadata storage only after the user confirms credentials, root access method, and whether remote access should be enabled. Do not run `mysql_secure_installation` non-interactively. +8. Install or verify etcd, then confirm endpoint health. +9. Update `pixels.properties` with the confirmed paths, hosts, ports, database settings, etcd settings, cache setting, and optional Trino JDBC URL. +10. Start Pixels only when the user asks to start services or when startup is part of the requested installation task. +11. Run smoke checks after configuration or startup changes. + +### Optional components + +Do not install these unless the user explicitly asks for the corresponding later stage: + +- Trino or another query engine. +- Hadoop or HDFS configuration. +- AWS credentials. +- Prometheus, Grafana, node exporter, or JMX exporter. +- Pixels Cache, Turbo, Retina, or Amphi. + +For Trino, `docs/INSTALL.md` points to the external `pixels-trino` documentation. The basic Pixels agent may configure `presto.pixels.jdbc.url` only after the Trino endpoint and catalog/schema values are known. + +### Examples + +```bash +/home/ubuntu/pixels/agents/pixels-install/scripts/prepare_deployment.sh \ + --ssh-user root \ + --setup-ssh true \ + --node 10.0.0.11,10.0.0.11,pixels-coordinator,root,22 \ + --node 10.0.0.12,10.0.0.12,pixels-worker-1,root,22 \ + --node 10.0.0.13,10.0.0.13,pixels-worker-2,root,22 +``` + +```bash +/home/ubuntu/pixels/agents/pixels-install/scripts/check_prerequisites.sh +``` + +### Guardrails + +- Prefer `docs/INSTALL.md` plus targeted helper scripts over a fixed all-in-one install command. +- Do not run scripts that install packages, change services, update credentials, or start daemons unless the user is asking to perform that installation stage. +- Do not modify Git state. +- Keep changes limited to `agents/pixels-install/` and the shared `agents/scripts/setup_cluster.sh` unless the user explicitly requests more. +- Ask before handling secrets, installing downloaded JDK packages, changing MySQL root authentication, or enabling remote database access. diff --git a/agents/pixels-install/scripts/build_install_pixels.sh b/agents/pixels-install/scripts/build_install_pixels.sh new file mode 100755 index 000000000..93f624c9f --- /dev/null +++ b/agents/pixels-install/scripts/build_install_pixels.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)}" +PIXELS_HOME="${PIXELS_HOME:-$HOME/opt/pixels}" +PROFILE_FILE="${PROFILE_FILE:-$HOME/.bashrc}" +MYSQL_CONNECTOR_VERSION="${MYSQL_CONNECTOR_VERSION:-8.0.33}" +MYSQL_CONNECTOR_JAR="${MYSQL_CONNECTOR_JAR:-$PIXELS_HOME/lib/mysql-connector-j-$MYSQL_CONNECTOR_VERSION.jar}" +MYSQL_CONNECTOR_URL="${MYSQL_CONNECTOR_URL:-https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/$MYSQL_CONNECTOR_VERSION/mysql-connector-j-$MYSQL_CONNECTOR_VERSION.jar}" +AUTO_CONFIRM_INSTALL="${AUTO_CONFIRM_INSTALL:-true}" + +log() { + printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*" +} + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "$1 command not found" +} + +persist_pixels_home() { + local line + + mkdir -p "$PIXELS_HOME" + line="export PIXELS_HOME=$PIXELS_HOME" + touch "$PROFILE_FILE" + + if ! grep -qxF "$line" "$PROFILE_FILE" 2>/dev/null; then + log "persisting PIXELS_HOME in $PROFILE_FILE" + printf '\n%s\n' "$line" >> "$PROFILE_FILE" + fi + + export PIXELS_HOME +} + +run_pixels_install() { + [[ -x "$REPO_ROOT/install.sh" ]] || fail "install.sh not found or not executable: $REPO_ROOT/install.sh" + require_command mvn + + log "installing Pixels from $REPO_ROOT to $PIXELS_HOME" + if [[ "$AUTO_CONFIRM_INSTALL" == "true" ]]; then + yes y | (cd "$REPO_ROOT" && ./install.sh) + else + (cd "$REPO_ROOT" && ./install.sh) + fi +} + +install_mysql_connector() { + local existing + + mkdir -p "$PIXELS_HOME/lib" + + existing="$(find "$PIXELS_HOME/lib" -maxdepth 1 -name 'mysql-connector-j-*.jar' -print -quit 2>/dev/null || true)" + if [[ -n "$existing" ]]; then + log "MySQL JDBC connector already exists: $existing" + return + fi + + if [[ -f "$MYSQL_CONNECTOR_JAR" ]]; then + log "MySQL JDBC connector already exists: $MYSQL_CONNECTOR_JAR" + return + fi + + require_command curl + log "downloading MySQL JDBC connector $MYSQL_CONNECTOR_VERSION" + curl -fL "$MYSQL_CONNECTOR_URL" -o "$MYSQL_CONNECTOR_JAR" +} + +verify_install() { + [[ -d "$PIXELS_HOME/bin" ]] || fail "missing directory: $PIXELS_HOME/bin" + [[ -d "$PIXELS_HOME/sbin" ]] || fail "missing directory: $PIXELS_HOME/sbin" + [[ -d "$PIXELS_HOME/etc" ]] || fail "missing directory: $PIXELS_HOME/etc" + [[ -d "$PIXELS_HOME/lib" ]] || fail "missing directory: $PIXELS_HOME/lib" + compgen -G "$PIXELS_HOME/bin/pixels-daemon-*-full.jar" >/dev/null || fail "pixels daemon jar not found in $PIXELS_HOME/bin" + compgen -G "$PIXELS_HOME/sbin/pixels-cli-*-full.jar" >/dev/null || fail "pixels cli jar not found in $PIXELS_HOME/sbin" + [[ -f "$PIXELS_HOME/etc/pixels.properties" ]] || fail "missing config: $PIXELS_HOME/etc/pixels.properties" + find "$PIXELS_HOME/lib" -maxdepth 1 -name 'mysql-connector-j-*.jar' -print -quit | grep -q . || fail "MySQL JDBC connector not found in $PIXELS_HOME/lib" + + log "Pixels installation verified at $PIXELS_HOME" +} + +main() { + persist_pixels_home + run_pixels_install + install_mysql_connector + verify_install +} + +main "$@" diff --git a/agents/pixels-install/scripts/check_prerequisites.sh b/agents/pixels-install/scripts/check_prerequisites.sh new file mode 100755 index 000000000..43fbc6833 --- /dev/null +++ b/agents/pixels-install/scripts/check_prerequisites.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +set -euo pipefail + +MIN_MEMORY_MB="${MIN_MEMORY_MB:-4096}" +MIN_DISK_GB="${MIN_DISK_GB:-20}" +CHECK_PORTS="${CHECK_PORTS:-18888 18889 18893 2379 3306}" +CHECK_HOSTS="${CHECK_HOSTS:-}" +CHECK_SSH_HOSTS="${CHECK_SSH_HOSTS:-}" +SSH_USER="${SSH_USER:-}" +SSH_PORT="${SSH_PORT:-}" + +log() { + printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*" +} + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +warn() { + printf 'WARN: %s\n' "$*" >&2 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1" +} + +check_os() { + [[ -r /etc/os-release ]] || fail "/etc/os-release not found" + . /etc/os-release + + if [[ "${ID:-}" != "ubuntu" ]]; then + warn "expected Ubuntu, found '${PRETTY_NAME:-unknown}'" + else + log "OS: ${PRETTY_NAME:-Ubuntu}" + fi + + arch="$(uname -m)" + if [[ "$arch" != "x86_64" && "$arch" != "amd64" ]]; then + warn "expected x86_64 architecture, found '$arch'" + else + log "architecture: $arch" + fi +} + +check_memory() { + mem_mb="$(awk '/MemTotal/ { print int($2 / 1024) }' /proc/meminfo)" + [[ -n "$mem_mb" ]] || fail "could not read system memory" + + if (( mem_mb < MIN_MEMORY_MB )); then + fail "memory is ${mem_mb}MB, expected at least ${MIN_MEMORY_MB}MB" + fi + + log "memory: ${mem_mb}MB" +} + +check_disk() { + disk_gb="$(df -BG / | awk 'NR == 2 { gsub(/G/, "", $4); print $4 }')" + [[ -n "$disk_gb" ]] || fail "could not read free disk space for /" + + if (( disk_gb < MIN_DISK_GB )); then + fail "free disk on / is ${disk_gb}GB, expected at least ${MIN_DISK_GB}GB" + fi + + log "free disk on /: ${disk_gb}GB" +} + +check_privilege() { + if [[ "$(id -u)" -eq 0 ]]; then + log "privilege: running as root" + return + fi + + if command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then + log "privilege: passwordless sudo is available" + return + fi + + fail "run as root or configure passwordless sudo for this user" +} + +check_ports() { + local port + + require_command ss + for port in $CHECK_PORTS; do + [[ "$port" =~ ^[0-9]+$ ]] || fail "invalid port in CHECK_PORTS: $port" + if ss -ltn "sport = :$port" | awk 'NR > 1 { found = 1 } END { exit(found ? 0 : 1) }'; then + fail "port $port is already listening" + fi + log "port available: $port" + done +} + +check_hosts() { + local host + + [[ -z "$CHECK_HOSTS" ]] && return + require_command getent + + for host in $CHECK_HOSTS; do + getent hosts "$host" >/dev/null || fail "host cannot be resolved: $host" + log "host resolved: $host" + done +} + +ssh_target() { + local host="$1" + if [[ -n "$SSH_USER" && "$host" != *@* ]]; then + printf '%s@%s' "$SSH_USER" "$host" + else + printf '%s' "$host" + fi +} + +check_ssh_hosts() { + local host + local -a ssh_args + + [[ -z "$CHECK_SSH_HOSTS" ]] && return + require_command ssh + + ssh_args=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new) + if [[ -n "$SSH_PORT" ]]; then + ssh_args+=(-p "$SSH_PORT") + fi + + for host in $CHECK_SSH_HOSTS; do + ssh "${ssh_args[@]}" "$(ssh_target "$host")" true >/dev/null 2>&1 || fail "SSH check failed: $host" + log "SSH ok: $host" + done +} + +main() { + log "checking prerequisites" + require_command awk + require_command df + require_command uname + + check_os + check_memory + check_disk + check_privilege + check_ports + check_hosts + check_ssh_hosts + + log "prerequisite checks passed" +} + +main "$@" diff --git a/agents/pixels-install/scripts/configure_pixels.sh b/agents/pixels-install/scripts/configure_pixels.sh new file mode 100755 index 000000000..704894f26 --- /dev/null +++ b/agents/pixels-install/scripts/configure_pixels.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +set -euo pipefail + +PIXELS_HOME="${PIXELS_HOME:-$HOME/opt/pixels}" +CONFIG_FILE="${PIXELS_CONFIG_FILE:-$PIXELS_HOME/etc/pixels.properties}" +BACKUP_FILE="${BACKUP_FILE:-$CONFIG_FILE.bak.$(date '+%Y%m%d%H%M%S')}" + +PIXELS_VAR_DIR="${PIXELS_VAR_DIR:-$PIXELS_HOME/var/}" +METADATA_DB_DRIVER="${METADATA_DB_DRIVER:-com.mysql.cj.jdbc.Driver}" +METADATA_DB_USER="${METADATA_DB_USER:-pixels}" +METADATA_DB_PASSWORD="${METADATA_DB_PASSWORD:-password}" +METADATA_DB_HOST="${METADATA_DB_HOST:-localhost}" +METADATA_DB_PORT="${METADATA_DB_PORT:-3306}" +METADATA_DB_NAME="${METADATA_DB_NAME:-pixels_metadata}" +METADATA_DB_URL="${METADATA_DB_URL:-jdbc:mysql://$METADATA_DB_HOST:$METADATA_DB_PORT/$METADATA_DB_NAME?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull}" +METADATA_SERVER_HOST="${METADATA_SERVER_HOST:-localhost}" +METADATA_SERVER_PORT="${METADATA_SERVER_PORT:-18888}" +TRANS_SERVER_HOST="${TRANS_SERVER_HOST:-localhost}" +TRANS_SERVER_PORT="${TRANS_SERVER_PORT:-18889}" +QUERY_SCHEDULE_SERVER_HOST="${QUERY_SCHEDULE_SERVER_HOST:-localhost}" +QUERY_SCHEDULE_SERVER_PORT="${QUERY_SCHEDULE_SERVER_PORT:-18893}" +ETCD_HOSTS="${ETCD_HOSTS:-localhost}" +ETCD_PORT="${ETCD_PORT:-2379}" +METRICS_NODE_TEXT_DIR="${METRICS_NODE_TEXT_DIR:-$HOME/opt/node_exporter/text/}" +PRESTO_PIXELS_JDBC_URL="${PRESTO_PIXELS_JDBC_URL:-jdbc:trino://localhost:8080/pixels/tpch}" +CACHE_ENABLED="${CACHE_ENABLED:-false}" + +log() { + printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*" +} + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +set_property() { + local key="$1" + local value="$2" + local escaped_key + local escaped_value + + escaped_key="$(printf '%s' "$key" | sed 's/[.[\\*^$()+?{}|]/\\&/g')" + escaped_value="$(printf '%s' "$value" | sed 's/[\\&]/\\&/g')" + + if grep -qE "^[[:space:]]*${escaped_key}=" "$CONFIG_FILE"; then + sed -i -E "s|^[[:space:]]*${escaped_key}=.*|${key}=${escaped_value}|" "$CONFIG_FILE" + else + printf '%s=%s\n' "$key" "$value" >> "$CONFIG_FILE" + fi +} + +validate_boolean() { + case "$CACHE_ENABLED" in + true|false) ;; + *) fail "CACHE_ENABLED must be true or false" ;; + esac +} + +validate_config_file() { + [[ -f "$CONFIG_FILE" ]] || fail "missing Pixels config file: $CONFIG_FILE" + [[ -w "$CONFIG_FILE" ]] || fail "Pixels config file is not writable: $CONFIG_FILE" +} + +configure_pixels() { + cp "$CONFIG_FILE" "$BACKUP_FILE" + log "backup created: $BACKUP_FILE" + + set_property pixels.var.dir "$PIXELS_VAR_DIR" + set_property metadata.db.driver "$METADATA_DB_DRIVER" + set_property metadata.db.user "$METADATA_DB_USER" + set_property metadata.db.password "$METADATA_DB_PASSWORD" + set_property metadata.db.url "$METADATA_DB_URL" + set_property metadata.server.port "$METADATA_SERVER_PORT" + set_property metadata.server.host "$METADATA_SERVER_HOST" + set_property trans.server.port "$TRANS_SERVER_PORT" + set_property trans.server.host "$TRANS_SERVER_HOST" + set_property query.schedule.server.port "$QUERY_SCHEDULE_SERVER_PORT" + set_property query.schedule.server.host "$QUERY_SCHEDULE_SERVER_HOST" + set_property etcd.hosts "$ETCD_HOSTS" + set_property etcd.port "$ETCD_PORT" + set_property metrics.node.text.dir "$METRICS_NODE_TEXT_DIR" + set_property presto.pixels.jdbc.url "$PRESTO_PIXELS_JDBC_URL" + set_property cache.enabled "$CACHE_ENABLED" +} + +verify_property() { + local key="$1" + local expected="$2" + local actual + + actual="$(grep -E "^[[:space:]]*${key}=" "$CONFIG_FILE" | tail -n 1 | cut -d= -f2-)" + [[ "$actual" == "$expected" ]] || fail "$key expected '$expected' but found '$actual'" +} + +verify_config() { + verify_property pixels.var.dir "$PIXELS_VAR_DIR" + verify_property metadata.db.driver "$METADATA_DB_DRIVER" + verify_property metadata.db.user "$METADATA_DB_USER" + verify_property metadata.db.password "$METADATA_DB_PASSWORD" + verify_property metadata.db.url "$METADATA_DB_URL" + verify_property metadata.server.port "$METADATA_SERVER_PORT" + verify_property metadata.server.host "$METADATA_SERVER_HOST" + verify_property trans.server.port "$TRANS_SERVER_PORT" + verify_property trans.server.host "$TRANS_SERVER_HOST" + verify_property query.schedule.server.port "$QUERY_SCHEDULE_SERVER_PORT" + verify_property query.schedule.server.host "$QUERY_SCHEDULE_SERVER_HOST" + verify_property etcd.hosts "$ETCD_HOSTS" + verify_property etcd.port "$ETCD_PORT" + verify_property metrics.node.text.dir "$METRICS_NODE_TEXT_DIR" + verify_property presto.pixels.jdbc.url "$PRESTO_PIXELS_JDBC_URL" + verify_property cache.enabled "$CACHE_ENABLED" + + log "Pixels config updated: $CONFIG_FILE" +} + +main() { + validate_boolean + validate_config_file + configure_pixels + verify_config +} + +main "$@" diff --git a/agents/pixels-install/scripts/install_etcd.sh b/agents/pixels-install/scripts/install_etcd.sh new file mode 100755 index 000000000..320287564 --- /dev/null +++ b/agents/pixels-install/scripts/install_etcd.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)}" +ETCD_VERSION="${ETCD_VERSION:-3.3.4}" +ETCD_ARCHIVE="${ETCD_ARCHIVE:-$REPO_ROOT/scripts/tars/etcd-v$ETCD_VERSION-linux-amd64.tar.xz}" +ETCD_INSTALL_PARENT="${ETCD_INSTALL_PARENT:-$HOME/opt}" +ETCD_INSTALL_DIR="${ETCD_INSTALL_DIR:-$ETCD_INSTALL_PARENT/etcd-v$ETCD_VERSION-linux-amd64-bin}" +ETCD_HOME_LINK="${ETCD_HOME_LINK:-$ETCD_INSTALL_PARENT/etcd}" +PROFILE_FILE="${PROFILE_FILE:-$HOME/.bashrc}" +START_ETCD="${START_ETCD:-true}" +ETCD_ENDPOINT="${ETCD_ENDPOINT:-http://127.0.0.1:2379}" + +log() { + printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*" +} + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +sudo_cmd() { + if [[ "$(id -u)" -eq 0 ]]; then + "$@" + elif command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then + sudo -n "$@" + else + fail "run as root or configure passwordless sudo for this user" + fi +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "$1 command not found" +} + +install_go_if_missing() { + if command -v go >/dev/null 2>&1; then + log "Go is already installed: $(go version)" + return + fi + + require_command apt-get + log "installing golang" + sudo_cmd apt-get update + sudo_cmd env DEBIAN_FRONTEND=noninteractive apt-get install -y golang +} + +install_etcd() { + require_command tar + [[ -f "$ETCD_ARCHIVE" ]] || fail "etcd archive not found: $ETCD_ARCHIVE" + + mkdir -p "$ETCD_INSTALL_PARENT" + + if [[ -d "$ETCD_INSTALL_DIR" ]]; then + log "etcd install directory already exists: $ETCD_INSTALL_DIR" + else + log "extracting etcd archive to $ETCD_INSTALL_PARENT" + tar -xJf "$ETCD_ARCHIVE" -C "$ETCD_INSTALL_PARENT" + fi + + [[ -x "$ETCD_INSTALL_DIR/etcd" ]] || fail "etcd binary not found: $ETCD_INSTALL_DIR/etcd" + [[ -x "$ETCD_INSTALL_DIR/etcdctl" ]] || fail "etcdctl binary not found: $ETCD_INSTALL_DIR/etcdctl" + + ln -sfn "$ETCD_INSTALL_DIR" "$ETCD_HOME_LINK" +} + +persist_environment() { + local line_api line_home line_path + + line_api='export ETCDCTL_API=3' + line_home="export ETCD=$ETCD_INSTALL_DIR" + line_path='export PATH=$PATH:$ETCD' + + touch "$PROFILE_FILE" + + if ! grep -qxF "$line_api" "$PROFILE_FILE" 2>/dev/null; then + log "persisting ETCDCTL_API in $PROFILE_FILE" + printf '\n%s\n' "$line_api" >> "$PROFILE_FILE" + fi + + if ! grep -qxF "$line_home" "$PROFILE_FILE" 2>/dev/null; then + log "persisting ETCD in $PROFILE_FILE" + printf '%s\n%s\n' "$line_home" "$line_path" >> "$PROFILE_FILE" + fi + + export ETCDCTL_API=3 + export ETCD="$ETCD_INSTALL_DIR" + export PATH="$PATH:$ETCD" +} + +start_etcd() { + [[ "$START_ETCD" == "true" ]] || { + log "etcd startup disabled" + return + } + + if "$ETCD_INSTALL_DIR/etcdctl" --endpoints="$ETCD_ENDPOINT" endpoint health >/dev/null 2>&1; then + log "etcd is already healthy at $ETCD_ENDPOINT" + return + fi + + if [[ -x "$ETCD_HOME_LINK/start-etcd.sh" ]]; then + log "starting etcd with $ETCD_HOME_LINK/start-etcd.sh" + (cd "$ETCD_HOME_LINK" && ./start-etcd.sh) + else + log "start-etcd.sh not found; starting etcd with default localhost settings" + nohup "$ETCD_INSTALL_DIR/etcd" >/tmp/pixels-etcd.log 2>&1 & + fi +} + +verify_etcd() { + local attempt + + for attempt in {1..20}; do + if "$ETCD_INSTALL_DIR/etcdctl" --endpoints="$ETCD_ENDPOINT" endpoint health >/dev/null 2>&1; then + log "etcd verification passed: $ETCD_ENDPOINT" + "$ETCD_INSTALL_DIR/etcdctl" --endpoints="$ETCD_ENDPOINT" endpoint health + return + fi + sleep 1 + done + + fail "etcd endpoint is not healthy: $ETCD_ENDPOINT" +} + +main() { + install_go_if_missing + install_etcd + persist_environment + start_etcd + verify_etcd +} + +main "$@" diff --git a/agents/pixels-install/scripts/install_maven.sh b/agents/pixels-install/scripts/install_maven.sh new file mode 100755 index 000000000..84538ca74 --- /dev/null +++ b/agents/pixels-install/scripts/install_maven.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail + +MIN_MAVEN_VERSION="${MIN_MAVEN_VERSION:-3.8.0}" +MAVEN_VERSION="${MAVEN_VERSION:-3.9.9}" +MAVEN_BASE_URL="${MAVEN_BASE_URL:-https://archive.apache.org/dist/maven/maven-3}" +MAVEN_INSTALL_DIR="${MAVEN_INSTALL_DIR:-/opt/apache-maven-$MAVEN_VERSION}" +MAVEN_HOME_LINK="${MAVEN_HOME_LINK:-/opt/maven}" +PROFILE_FILE="${PROFILE_FILE:-$HOME/.bashrc}" +TMP_DIR="${TMP_DIR:-/tmp}" + +log() { + printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*" +} + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +sudo_cmd() { + if [[ "$(id -u)" -eq 0 ]]; then + "$@" + elif command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then + sudo -n "$@" + else + fail "run as root or configure passwordless sudo for this user" + fi +} + +version_ge() { + local current="$1" + local required="$2" + + [[ "$(printf '%s\n%s\n' "$required" "$current" | sort -V | head -n1)" == "$required" ]] +} + +maven_version() { + command -v mvn >/dev/null 2>&1 || return 1 + mvn -v 2>/dev/null | awk '/Apache Maven/ { print $3; exit }' +} + +maven_satisfies_requirement() { + local version + + version="$(maven_version || true)" + [[ -n "$version" ]] || return 1 + version_ge "$version" "$MIN_MAVEN_VERSION" +} + +require_java() { + command -v java >/dev/null 2>&1 || fail "java command not found; install or select a JDK according to docs/INSTALL.md first" + [[ -n "${JAVA_HOME:-}" ]] || fail "JAVA_HOME is not set; configure JAVA_HOME for the selected JDK first" + [[ -x "$JAVA_HOME/bin/java" ]] || fail "JAVA_HOME does not contain bin/java: $JAVA_HOME" +} + +install_maven() { + local archive url + + command -v curl >/dev/null 2>&1 || fail "curl command not found; install curl first" + command -v tar >/dev/null 2>&1 || fail "tar command not found; install tar first" + + archive="$TMP_DIR/apache-maven-$MAVEN_VERSION-bin.tar.gz" + url="$MAVEN_BASE_URL/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz" + + if [[ ! -d "$MAVEN_INSTALL_DIR" ]]; then + log "downloading Maven $MAVEN_VERSION" + curl -fsSL "$url" -o "$archive" + + log "installing Maven to $MAVEN_INSTALL_DIR" + sudo_cmd tar -xzf "$archive" -C /opt + else + log "Maven install directory already exists: $MAVEN_INSTALL_DIR" + fi + + sudo_cmd ln -sfn "$MAVEN_INSTALL_DIR" "$MAVEN_HOME_LINK" +} + +persist_environment() { + local line_home line_path + + line_home="export MAVEN_HOME=$MAVEN_HOME_LINK" + line_path='export PATH=$MAVEN_HOME/bin:$PATH' + + if ! grep -qxF "$line_home" "$PROFILE_FILE" 2>/dev/null; then + log "persisting MAVEN_HOME in $PROFILE_FILE" + printf '\n%s\n%s\n' "$line_home" "$line_path" >> "$PROFILE_FILE" + fi + + export MAVEN_HOME="$MAVEN_HOME_LINK" + export PATH="$MAVEN_HOME/bin:$PATH" +} + +verify_maven() { + local version mvn_java_home + + command -v mvn >/dev/null 2>&1 || fail "mvn command not found" + version="$(maven_version || true)" + [[ -n "$version" ]] || fail "could not parse Maven version" + version_ge "$version" "$MIN_MAVEN_VERSION" || fail "Maven version is $version, expected at least $MIN_MAVEN_VERSION" + + mvn_java_home="$(mvn -v | awk -F': ' '/Java home/ { print $2; exit }')" + [[ -n "$mvn_java_home" ]] || fail "could not read Java home from mvn -v" + + log "Maven verification passed: version=$version, java_home=$mvn_java_home" + mvn -v +} + +main() { + require_java + + if maven_satisfies_requirement; then + log "existing Maven satisfies requirement: version=$(maven_version)" + else + install_maven + fi + + persist_environment + verify_maven +} + +main "$@" diff --git a/agents/pixels-install/scripts/prepare_deployment.sh b/agents/pixels-install/scripts/prepare_deployment.sh new file mode 100755 index 000000000..299e0d73d --- /dev/null +++ b/agents/pixels-install/scripts/prepare_deployment.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +set -euo pipefail + +AGENT_DIR="${AGENT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +REPO_ROOT="${REPO_ROOT:-$(cd "$AGENT_DIR/../.." && pwd)}" +SHARED_SETUP_CLUSTER="${SHARED_SETUP_CLUSTER:-$REPO_ROOT/agents/scripts/setup_cluster.sh}" +OUTPUT_FILE="${OUTPUT_FILE:-$AGENT_DIR/deployment.env}" +PIXELS_HOME="${PIXELS_HOME:-$HOME/opt/pixels}" +SSH_USER="${SSH_USER:-root}" +SSH_PORT="${SSH_PORT:-}" +VERIFY_REMOTE_LOGIN="${VERIFY_REMOTE_LOGIN:-true}" +SETUP_SSH="${SETUP_SSH:-false}" + +NODES=() + +usage() { + cat < [options] + $0 --nodes-file [options] + +Options: + --node + Add one deployment node. Can be repeated. The first node is treated as the + coordinator by default; all nodes are written to PIXELS_WORKERS. + + --nodes-file + Read deployment nodes from a CSV file. Each non-empty, non-comment line + must use: ssh_target,host_ip,host_name,host_user,host_port. + + --pixels-home + Pixels home on each node. Default: $HOME/opt/pixels + + --ssh-user + User used by the local machine when connecting to ssh_target values that + do not already include user@host. Default: root + + --ssh-port + Shared SSH port used by the local machine. Leave empty for SSH defaults. + + --setup-ssh + Run agents/scripts/setup_cluster.sh after writing deployment.env. + Default: false + + --verify-remote-login + Passed to setup_cluster.sh when --setup-ssh true. Default: true + + --output + Deployment env file path. Default: $AGENT_DIR/deployment.env + + -h, --help + Show this help message. +EOF +} + +log() { + printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*" +} + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +parse_bool() { + case "$1" in + true|false) printf '%s\n' "$1" ;; + *) fail "expected true or false, got: $1" ;; + esac +} + +validate_node_spec() { + local spec="$1" + local ssh_target host_ip host_name host_user host_port extra + + IFS=, read -r ssh_target host_ip host_name host_user host_port extra <<< "$spec" + [[ -z "${extra:-}" ]] || fail "--node expects exactly 5 comma-separated fields" + [[ -n "${ssh_target:-}" ]] || fail "--node ssh_target is empty" + [[ -n "${host_ip:-}" ]] || fail "--node host_ip is empty" + [[ -n "${host_name:-}" ]] || fail "--node host_name is empty" + [[ -n "${host_user:-}" ]] || fail "--node host_user is empty" + [[ "${host_port:-}" =~ ^[0-9]+$ ]] || fail "--node host_port must be a number" + (( host_port >= 1 && host_port <= 65535 )) || fail "--node host_port must be between 1 and 65535" +} + +add_node_arg() { + validate_node_spec "$1" + NODES+=("$1") +} + +add_nodes_file_arg() { + local file="$1" + local line + local node_count=0 + + [[ -f "$file" ]] || fail "--nodes-file not found: $file" + + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line%$'\r'}" + line="$(printf '%s' "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + [[ -n "$line" ]] || continue + [[ "${line:0:1}" != "#" ]] || continue + + add_node_arg "$line" + ((node_count += 1)) + done < "$file" + + [[ "$node_count" -gt 0 ]] || fail "--nodes-file has no nodes: $file" +} + +parse_args() { + while [[ "$#" -gt 0 ]]; do + case "$1" in + --node) + [[ "$#" -ge 2 ]] || fail "--node requires a value" + add_node_arg "$2" + shift 2 + ;; + --nodes-file) + [[ "$#" -ge 2 ]] || fail "--nodes-file requires a value" + add_nodes_file_arg "$2" + shift 2 + ;; + --pixels-home) + [[ "$#" -ge 2 ]] || fail "--pixels-home requires a value" + PIXELS_HOME="$2" + shift 2 + ;; + --ssh-user) + [[ "$#" -ge 2 ]] || fail "--ssh-user requires a value" + SSH_USER="$2" + shift 2 + ;; + --ssh-port) + [[ "$#" -ge 2 ]] || fail "--ssh-port requires a value" + SSH_PORT="$2" + shift 2 + ;; + --setup-ssh) + [[ "$#" -ge 2 ]] || fail "--setup-ssh requires a value" + SETUP_SSH="$(parse_bool "$2")" + shift 2 + ;; + --verify-remote-login) + [[ "$#" -ge 2 ]] || fail "--verify-remote-login requires a value" + VERIFY_REMOTE_LOGIN="$(parse_bool "$2")" + shift 2 + ;; + --output) + [[ "$#" -ge 2 ]] || fail "--output requires a value" + OUTPUT_FILE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac + done + + [[ "${#NODES[@]}" -gt 0 ]] || fail "at least one --node or --nodes-file entry is required" +} + +csv_field() { + local spec="$1" + local field_index="$2" + + awk -F, -v field_index="$field_index" '{ print $field_index }' <<< "$spec" +} + +join_by_space() { + local first=true + local value + + for value in "$@"; do + if [[ "$first" == "true" ]]; then + printf '%s' "$value" + first=false + else + printf ' %s' "$value" + fi + done +} + +shell_quote() { + printf '%q' "$1" +} + +write_deployment_env() { + local coordinator_host + local coordinator_ssh_target + local metadata_host + local workers=() + local workers_value + local node host_name + + coordinator_ssh_target="$(csv_field "${NODES[0]}" 1)" + coordinator_host="$(csv_field "${NODES[0]}" 3)" + metadata_host="$coordinator_host" + + for node in "${NODES[@]}"; do + host_name="$(csv_field "$node" 3)" + workers+=("$host_name") + done + workers_value="$(join_by_space "${workers[@]}")" + + mkdir -p "$(dirname "$OUTPUT_FILE")" + cat > "$OUTPUT_FILE" <&2 +} + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "$1 command not found" +} + +property_value() { + local key="$1" + + awk -F= -v key="$key" ' + $1 == key { + value = $0 + sub(/^[^=]*=/, "", value) + result = value + } + END { + if (result != "") { + print result + } + } + ' "$CONFIG_FILE" +} + +property_or_default() { + local key="$1" + local default_value="$2" + local value + + value="$(property_value "$key")" + if [[ -n "$value" ]]; then + printf '%s\n' "$value" + else + printf '%s\n' "$default_value" + fi +} + +first_glob_match() { + local pattern="$1" + local matches + + matches="$(compgen -G "$pattern" || true)" + [[ -n "$matches" ]] || return 1 + printf '%s\n' "$matches" | head -n 1 +} + +port_is_listening() { + local host="$1" + local port="$2" + + if command -v ss >/dev/null 2>&1; then + if [[ "$host" == "localhost" || "$host" == "127.0.0.1" || "$host" == "0.0.0.0" ]]; then + ss -ltn | awk '{print $4}' | grep -Eq "(^|:)${port}$" + return + fi + fi + + require_command nc + nc -z "$host" "$port" >/dev/null 2>&1 +} + +wait_for_port() { + local name="$1" + local host="$2" + local port="$3" + local deadline + + deadline=$((SECONDS + SMOKE_TIMEOUT_SECONDS)) + while (( SECONDS < deadline )); do + if port_is_listening "$host" "$port"; then + log "$name is reachable at $host:$port" + return + fi + sleep 2 + done + + fail "$name is not reachable at $host:$port within ${SMOKE_TIMEOUT_SECONDS}s" +} + +verify_filesystem() { + [[ -d "$PIXELS_HOME" ]] || fail "PIXELS_HOME does not exist: $PIXELS_HOME" + [[ -d "$PIXELS_HOME/bin" ]] || fail "missing directory: $PIXELS_HOME/bin" + [[ -d "$PIXELS_HOME/sbin" ]] || fail "missing directory: $PIXELS_HOME/sbin" + [[ -d "$PIXELS_HOME/etc" ]] || fail "missing directory: $PIXELS_HOME/etc" + [[ -f "$CONFIG_FILE" ]] || fail "missing Pixels config file: $CONFIG_FILE" + + first_glob_match "$PIXELS_HOME/bin/pixels-daemon-*-full.jar" >/dev/null || fail "missing pixels-daemon full jar in $PIXELS_HOME/bin" + first_glob_match "$PIXELS_HOME/sbin/pixels-cli-*-full.jar" >/dev/null || fail "missing pixels-cli full jar in $PIXELS_HOME/sbin" + + log "Pixels installation layout looks valid: $PIXELS_HOME" +} + +verify_config_property() { + local key="$1" + local value + + value="$(property_value "$key")" + [[ -n "$value" ]] || fail "missing or empty config property: $key" + log "config property found: $key=$value" +} + +verify_config() { + verify_config_property pixels.var.dir + verify_config_property metadata.db.driver + verify_config_property metadata.db.user + verify_config_property metadata.db.password + verify_config_property metadata.db.url + verify_config_property metadata.server.host + verify_config_property metadata.server.port + verify_config_property query.schedule.server.host + verify_config_property query.schedule.server.port + verify_config_property etcd.hosts + verify_config_property etcd.port + verify_config_property cache.enabled +} + +verify_java() { + require_command java + java -version >/dev/null 2>&1 || fail "java is installed but cannot run" + log "Java runtime is available" +} + +verify_core_services() { + local metadata_host + local query_schedule_host + local trans_host + local metadata_port + local query_schedule_port + local trans_port + + metadata_host="$(property_or_default metadata.server.host localhost)" + query_schedule_host="$(property_or_default query.schedule.server.host localhost)" + trans_host="$(property_or_default trans.server.host localhost)" + metadata_port="$(property_or_default metadata.server.port "$METADATA_SERVER_PORT")" + query_schedule_port="$(property_or_default query.schedule.server.port "$QUERY_SCHEDULE_SERVER_PORT")" + trans_port="$(property_or_default trans.server.port "$TRANS_SERVER_PORT")" + + wait_for_port "metadata server" "$metadata_host" "$metadata_port" + wait_for_port "query schedule server" "$query_schedule_host" "$query_schedule_port" + + if [[ "$CHECK_TRANS_SERVER" == "true" ]]; then + wait_for_port "transaction server" "$trans_host" "$trans_port" + else + log "transaction server port check skipped; set CHECK_TRANS_SERVER=true to enable it" + fi +} + +verify_etcd() { + local etcd_hosts + local etcd_host + local etcd_port + + if [[ "$CHECK_ETCD" != "true" ]]; then + log "etcd check skipped" + return + fi + + etcd_hosts="$(property_or_default etcd.hosts localhost)" + etcd_host="${etcd_hosts%%,*}" + etcd_host="${etcd_host%%;*}" + etcd_host="${etcd_host%%:*}" + etcd_port="$(property_or_default etcd.port "$ETCD_PORT")" + + wait_for_port "etcd" "$etcd_host" "$etcd_port" +} + +verify_mysql() { + local metadata_db_url + local mysql_host + local mysql_port + + if [[ "$CHECK_MYSQL" != "true" ]]; then + log "MySQL check skipped" + return + fi + + metadata_db_url="$(property_value metadata.db.url)" + mysql_host="$(printf '%s\n' "$metadata_db_url" | sed -nE 's#^jdbc:mysql://([^:/?]+).*#\1#p')" + mysql_port="$(printf '%s\n' "$metadata_db_url" | sed -nE 's#^jdbc:mysql://[^:/?]+:([0-9]+).*#\1#p')" + mysql_host="${mysql_host:-localhost}" + mysql_port="${mysql_port:-$MYSQL_PORT}" + + wait_for_port "MySQL" "$mysql_host" "$mysql_port" +} + +verify_pixels_cli() { + local cli_jar + + if [[ "$CHECK_PIXELS_CLI" != "true" ]]; then + log "Pixels CLI check skipped" + return + fi + + cli_jar="$(first_glob_match "$PIXELS_HOME/sbin/pixels-cli-*-full.jar")" + timeout 10s java -jar "$cli_jar" --help >/dev/null 2>&1 || { + warn "pixels-cli --help did not complete successfully; jar exists but interactive CLI may not support --help" + return + } + + log "Pixels CLI jar can be executed" +} + +main() { + export PIXELS_HOME + verify_filesystem + verify_config + verify_java + verify_core_services + verify_etcd + verify_mysql + verify_pixels_cli + log "Pixels smoke test passed" +} + +main "$@" diff --git a/agents/scripts/README.md b/agents/scripts/README.md new file mode 100644 index 000000000..9d5b8904c --- /dev/null +++ b/agents/scripts/README.md @@ -0,0 +1,5 @@ +## Shared Scripts + +This directory contains runtime scripts that can be reused by multiple agents. + +Agent-specific scripts should live under `agents//scripts/` instead. diff --git a/agents/scripts/setup_cluster.sh b/agents/scripts/setup_cluster.sh new file mode 100755 index 000000000..79ad8418b --- /dev/null +++ b/agents/scripts/setup_cluster.sh @@ -0,0 +1,732 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run this script on your local machine. It logs in to each remote server, +# ensures an SSH key exists, updates /etc/hosts, and distributes public keys. + +SSH_USER="${SSH_USER:-root}" # Set to empty if SSH_TARGETS already contains user@host or uses ssh config User. +SSH_PORT="${SSH_PORT:-}" # Leave empty to use ssh config / default port 22. +VERIFY_REMOTE_LOGIN="${VERIFY_REMOTE_LOGIN:-true}" + +# SSH_TARGETS: +# Used only by this local script to connect to each server. Each item can be +# an IP, DNS name, Host alias from ~/.ssh/config, or user@host. +SSH_TARGETS=( + "10.0.0.11" + "10.0.0.12" + "10.0.0.13" +) + +# HOSTS_IPS: +# Written into every remote server's /etc/hosts. Usually these are private +# or internal IPs. Keep the order aligned with SSH_TARGETS and HOST_NAMES. +HOSTS_IPS=( + "10.0.0.11" + "10.0.0.12" + "10.0.0.13" +) + +# HOST_NAMES: +# Written into /etc/hosts and used when verifying remote-to-remote SSH login. +# Keep the order aligned with SSH_TARGETS and HOSTS_IPS. +HOST_NAMES=( + "trino-coordinator" + "trino-worker-1" + "trino-worker-2" +) + +# HOST_USERS: +# Usernames used when cluster hosts SSH to each other. Keep the order aligned +# with SSH_TARGETS, HOSTS_IPS, and HOST_NAMES. +HOST_USERS=( + "root" + "root" + "root" +) + +# HOST_PORTS: +# SSH ports used when cluster hosts SSH to each other. Keep the order aligned +# with SSH_TARGETS, HOSTS_IPS, HOST_NAMES, and HOST_USERS. +HOST_PORTS=( + "22" + "22" + "22" +) + +MANAGED_BEGIN="# BEGIN managed by setup_cluster_ssh.sh" +MANAGED_END="# END managed by setup_cluster_ssh.sh" + +SSH_COMMON_OPTS=( + -o BatchMode=yes + -o ConnectTimeout=10 + -o StrictHostKeyChecking=accept-new +) + +WORKDIR="" +CUSTOM_NODE_CONFIG_ACTIVE=false +NODE_INPUT_MODE="" + +usage() { + cat < + Add one cluster node. Can be repeated. When at least one custom node input + option is used, the built-in SSH_TARGETS, HOSTS_IPS, HOST_NAMES, + HOST_USERS, and HOST_PORTS defaults are replaced. + + --nodes-file + Read cluster nodes from a CSV file. Each non-empty, non-comment line must + use: ssh_target,host_ip,host_name,host_user,host_port. + + --ssh-targets + Set SSH_TARGETS. Values may be space- or comma-separated. + + --hosts-ips + Set HOSTS_IPS. Values may be space- or comma-separated. + + --host-names + Set HOST_NAMES. Values may be space- or comma-separated. + + --host-users + Set HOST_USERS. Values may be space- or comma-separated. + + --host-ports + Set HOST_PORTS. Values may be space- or comma-separated. + + --ssh-user + User used by this local script when SSH_TARGETS do not include user@host. + Use an empty value to rely on SSH config or explicit user@host targets. + + --ssh-port + Port used by this local script when connecting to every SSH_TARGET. + Leave unset to use SSH config or default port 22. + + --verify-remote-login + Whether to verify passwordless SSH from each cluster node to the others. + + -h, --help + Show this help message. + +Examples: + $0 \ + --ssh-user root \ + --node 10.0.0.11,10.0.0.11,trino-coordinator,root,22 \ + --node 10.0.0.12,10.0.0.12,trino-worker-1,root,22 + + $0 \ + --ssh-targets "10.0.0.11 10.0.0.12" \ + --hosts-ips "10.0.0.11 10.0.0.12" \ + --host-names "trino-coordinator trino-worker-1" \ + --host-users "root root" \ + --host-ports "22 22" +EOF +} + +parse_bool() { + case "$1" in + true|false) printf '%s\n' "$1" ;; + *) die "expected true or false, got: $1" ;; + esac +} + +clear_node_config() { + SSH_TARGETS=() + HOSTS_IPS=() + HOST_NAMES=() + HOST_USERS=() + HOST_PORTS=() +} + +activate_custom_node_config() { + if [[ "$CUSTOM_NODE_CONFIG_ACTIVE" == "false" ]]; then + clear_node_config + CUSTOM_NODE_CONFIG_ACTIVE=true + fi +} + +use_node_input_mode() { + local mode="$1" + local option_name="$2" + + if [[ -n "$NODE_INPUT_MODE" && "$NODE_INPUT_MODE" != "$mode" ]]; then + die "$option_name cannot be mixed with $NODE_INPUT_MODE-based node options" + fi + + NODE_INPUT_MODE="$mode" + activate_custom_node_config +} + +set_array_arg() { + local array_name="$1" + local option_name="$2" + local raw_value="$3" + local normalized + local -a parsed=() + local -n values_ref="$array_name" + + normalized="${raw_value//,/ }" + read -r -a parsed <<< "$normalized" + [[ "${#parsed[@]}" -gt 0 ]] || die "$option_name requires at least one value" + + values_ref=("${parsed[@]}") +} + +add_node_arg() { + local spec="$1" + local ssh_target host_ip host_name host_user host_port extra + + IFS=, read -r ssh_target host_ip host_name host_user host_port extra <<< "$spec" + [[ -z "${extra:-}" ]] || die "--node expects exactly 5 comma-separated fields" + [[ -n "${ssh_target:-}" ]] || die "--node ssh_target is empty" + [[ -n "${host_ip:-}" ]] || die "--node host_ip is empty" + [[ -n "${host_name:-}" ]] || die "--node host_name is empty" + [[ -n "${host_user:-}" ]] || die "--node host_user is empty" + [[ -n "${host_port:-}" ]] || die "--node host_port is empty" + + SSH_TARGETS+=("$ssh_target") + HOSTS_IPS+=("$host_ip") + HOST_NAMES+=("$host_name") + HOST_USERS+=("$host_user") + HOST_PORTS+=("$host_port") +} + +add_nodes_file_arg() { + local file="$1" + local line + local node_count=0 + + [[ -f "$file" ]] || die "--nodes-file not found: $file" + + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line%$'\r'}" + line="$(printf '%s' "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + [[ -n "$line" ]] || continue + [[ "${line:0:1}" != "#" ]] || continue + + add_node_arg "$line" + ((node_count += 1)) + done < "$file" + + [[ "$node_count" -gt 0 ]] || die "--nodes-file has no nodes: $file" +} + +parse_args() { + while [[ "$#" -gt 0 ]]; do + case "$1" in + --node) + [[ "$#" -ge 2 ]] || die "--node requires a value" + use_node_input_mode "node" "--node" + add_node_arg "$2" + shift 2 + ;; + --nodes-file) + [[ "$#" -ge 2 ]] || die "--nodes-file requires a value" + use_node_input_mode "node" "--nodes-file" + add_nodes_file_arg "$2" + shift 2 + ;; + --ssh-targets) + [[ "$#" -ge 2 ]] || die "--ssh-targets requires a value" + use_node_input_mode "array" "--ssh-targets" + set_array_arg SSH_TARGETS "--ssh-targets" "$2" + shift 2 + ;; + --hosts-ips) + [[ "$#" -ge 2 ]] || die "--hosts-ips requires a value" + use_node_input_mode "array" "--hosts-ips" + set_array_arg HOSTS_IPS "--hosts-ips" "$2" + shift 2 + ;; + --host-names) + [[ "$#" -ge 2 ]] || die "--host-names requires a value" + use_node_input_mode "array" "--host-names" + set_array_arg HOST_NAMES "--host-names" "$2" + shift 2 + ;; + --host-users) + [[ "$#" -ge 2 ]] || die "--host-users requires a value" + use_node_input_mode "array" "--host-users" + set_array_arg HOST_USERS "--host-users" "$2" + shift 2 + ;; + --host-ports) + [[ "$#" -ge 2 ]] || die "--host-ports requires a value" + use_node_input_mode "array" "--host-ports" + set_array_arg HOST_PORTS "--host-ports" "$2" + shift 2 + ;; + --ssh-user) + [[ "$#" -ge 2 ]] || die "--ssh-user requires a value" + SSH_USER="$2" + shift 2 + ;; + --ssh-port) + [[ "$#" -ge 2 ]] || die "--ssh-port requires a value" + SSH_PORT="$2" + shift 2 + ;; + --verify-remote-login) + [[ "$#" -ge 2 ]] || die "--verify-remote-login requires a value" + VERIFY_REMOTE_LOGIN="$(parse_bool "$2")" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown argument: $1" + ;; + esac + done +} + +cleanup() { + if [[ -n "${WORKDIR:-}" && -d "$WORKDIR" ]]; then + rm -rf "$WORKDIR" + fi +} + +log() { + printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*" >&2 +} + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +remote_spec() { + local target="$1" + if [[ -n "$SSH_USER" && "$target" != *@* ]]; then + printf '%s@%s' "$SSH_USER" "$target" + else + printf '%s' "$target" + fi +} + +run_ssh() { + local target="$1" + shift + local -a args=("${SSH_COMMON_OPTS[@]}") + if [[ -n "$SSH_PORT" ]]; then + args+=(-p "$SSH_PORT") + fi + ssh "${args[@]}" "$(remote_spec "$target")" "$@" +} + +run_scp_to_remote() { + local source_file="$1" + local target="$2" + local remote_file="$3" + local -a args=("${SSH_COMMON_OPTS[@]}") + if [[ -n "$SSH_PORT" ]]; then + args+=(-P "$SSH_PORT") + fi + scp "${args[@]}" "$source_file" "$(remote_spec "$target"):$remote_file" +} + +validate_config() { + local count="${#SSH_TARGETS[@]}" + local i + + [[ "$count" -gt 0 ]] || die "SSH_TARGETS is empty" + [[ "${#HOSTS_IPS[@]}" -eq "$count" ]] || die "HOSTS_IPS count must match SSH_TARGETS count" + [[ "${#HOST_NAMES[@]}" -eq "$count" ]] || die "HOST_NAMES count must match SSH_TARGETS count" + [[ "${#HOST_USERS[@]}" -eq "$count" ]] || die "HOST_USERS count must match SSH_TARGETS count" + [[ "${#HOST_PORTS[@]}" -eq "$count" ]] || die "HOST_PORTS count must match SSH_TARGETS count" + + for ((i = 0; i < count; i++)); do + [[ -n "${SSH_TARGETS[$i]:-}" ]] || die "SSH_TARGETS[$i] is empty" + [[ -n "${HOSTS_IPS[$i]:-}" ]] || die "HOSTS_IPS[$i] is empty" + [[ -n "${HOST_NAMES[$i]:-}" ]] || die "HOST_NAMES[$i] is empty" + [[ -n "${HOST_USERS[$i]:-}" ]] || die "HOST_USERS[$i] is empty" + [[ "${HOST_PORTS[$i]:-}" =~ ^[0-9]+$ ]] || die "HOST_PORTS[$i] must be a number" + (( HOST_PORTS[$i] >= 1 && HOST_PORTS[$i] <= 65535 )) || die "HOST_PORTS[$i] must be between 1 and 65535" + done +} + +check_local_connectivity() { + local count="${#SSH_TARGETS[@]}" + local i target hostname + local -a failed_servers=() + + log "checking local SSH connectivity" + for ((i = 0; i < count; i++)); do + target="${SSH_TARGETS[$i]}" + hostname="${HOST_NAMES[$i]}" + + if run_ssh "$target" true >/dev/null 2>&1; then + log "local SSH OK: $hostname" + else + failed_servers+=("$hostname (target: $target, ssh: $(remote_spec "$target"))") + fi + done + + if [[ "${#failed_servers[@]}" -gt 0 ]]; then + printf '\nLocal SSH connection failed for these servers:\n' >&2 + for item in "${failed_servers[@]}"; do + printf ' - %s\n' "$item" >&2 + done + return 1 + fi +} + +ensure_remote_key_and_print_pubkey() { + local target="$1" + local remote_user="$2" + + run_ssh "$target" 'bash -s' -- "$remote_user" <<'REMOTE' +set -euo pipefail + +remote_user="$1" + +if [[ "$(id -un)" == "$remote_user" ]]; then + SUDO=() +elif [[ "$(id -u)" -eq 0 ]]; then + SUDO=() +elif command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then + SUDO=(sudo -n) +else + echo "Need root or passwordless sudo to manage SSH files for user '$remote_user' on $(hostname)" >&2 + exit 1 +fi + +home_dir="$(awk -F: -v user="$remote_user" '$1 == user { print $6; exit }' /etc/passwd)" +if [[ -z "$home_dir" ]]; then + echo "User '$remote_user' does not exist on $(hostname)" >&2 + exit 1 +fi + +run_as_remote_user() { + if [[ "$(id -un)" == "$remote_user" || "$(id -u)" -eq 0 ]]; then + "$@" + else + "${SUDO[@]}" -u "$remote_user" "$@" + fi +} + +"${SUDO[@]}" mkdir -p "$home_dir/.ssh" +"${SUDO[@]}" chmod 700 "$home_dir/.ssh" +"${SUDO[@]}" chown "$remote_user" "$home_dir/.ssh" 2>/dev/null || true + +pubkey="" +if [[ -s "$home_dir/.ssh/id_ed25519" && -s "$home_dir/.ssh/id_ed25519.pub" ]]; then + pubkey="$home_dir/.ssh/id_ed25519.pub" +elif [[ -s "$home_dir/.ssh/id_rsa" && -s "$home_dir/.ssh/id_rsa.pub" ]]; then + pubkey="$home_dir/.ssh/id_rsa.pub" +else + command -v ssh-keygen >/dev/null 2>&1 || { + echo "ssh-keygen not found on $(hostname)" >&2 + exit 1 + } + + echo "creating ed25519 key for $remote_user on $(hostname)" >&2 + run_as_remote_user ssh-keygen -q -t ed25519 -N "" -f "$home_dir/.ssh/id_ed25519" -C "$remote_user@$(hostname)" + pubkey="$home_dir/.ssh/id_ed25519.pub" +fi + +"${SUDO[@]}" chmod 600 "$home_dir/.ssh/id_ed25519" "$home_dir/.ssh/id_rsa" 2>/dev/null || true +"${SUDO[@]}" chmod 644 "$home_dir/.ssh/id_ed25519.pub" "$home_dir/.ssh/id_rsa.pub" 2>/dev/null || true +"${SUDO[@]}" chown "$remote_user" "$home_dir/.ssh/id_ed25519" "$home_dir/.ssh/id_ed25519.pub" "$home_dir/.ssh/id_rsa" "$home_dir/.ssh/id_rsa.pub" 2>/dev/null || true + +"${SUDO[@]}" cat "$pubkey" +REMOTE +} + +install_hosts_keys_and_known_hosts() { + local target="$1" + local local_hosts_file="$2" + local local_pubkeys_file="$3" + local local_ssh_config_file="$4" + local remote_user="$5" + local remote_hosts_file="/tmp/setup_cluster_hosts.$$" + local remote_pubkeys_file="/tmp/setup_cluster_pubkeys.$$" + local remote_ssh_config_file="/tmp/setup_cluster_ssh_config.$$" + + run_scp_to_remote "$local_hosts_file" "$target" "$remote_hosts_file" >/dev/null + run_scp_to_remote "$local_pubkeys_file" "$target" "$remote_pubkeys_file" >/dev/null + run_scp_to_remote "$local_ssh_config_file" "$target" "$remote_ssh_config_file" >/dev/null + + run_ssh "$target" 'bash -s' -- "$remote_hosts_file" "$remote_pubkeys_file" "$remote_ssh_config_file" "$remote_user" <<'REMOTE' +set -euo pipefail + +remote_hosts_file="$1" +remote_pubkeys_file="$2" +remote_ssh_config_file="$3" +remote_user="$4" +managed_begin="# BEGIN managed by setup_cluster_ssh.sh" +managed_end="# END managed by setup_cluster_ssh.sh" + +if [[ "$(id -u)" -eq 0 ]]; then + SUDO=() +elif command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then + SUDO=(sudo -n) +else + echo "Need root or passwordless sudo to update /etc/hosts and manage SSH files for user '$remote_user' on $(hostname)" >&2 + exit 1 +fi + +home_dir="$(awk -F: -v user="$remote_user" '$1 == user { print $6; exit }' /etc/passwd)" +if [[ -z "$home_dir" ]]; then + echo "User '$remote_user' does not exist on $(hostname)" >&2 + exit 1 +fi + +"${SUDO[@]}" mkdir -p "$home_dir/.ssh" +"${SUDO[@]}" chmod 700 "$home_dir/.ssh" +"${SUDO[@]}" chown "$remote_user" "$home_dir/.ssh" 2>/dev/null || true +"${SUDO[@]}" touch "$home_dir/.ssh/authorized_keys" +"${SUDO[@]}" chmod 600 "$home_dir/.ssh/authorized_keys" +"${SUDO[@]}" chown "$remote_user" "$home_dir/.ssh/authorized_keys" 2>/dev/null || true + +while IFS= read -r key; do + [[ -n "$key" ]] || continue + case "$key" in + ssh-ed25519\ *|ssh-rsa\ *) ;; + *) continue ;; + esac + "${SUDO[@]}" grep -qxF "$key" "$home_dir/.ssh/authorized_keys" || + printf '%s\n' "$key" | "${SUDO[@]}" tee -a "$home_dir/.ssh/authorized_keys" >/dev/null +done < "$remote_pubkeys_file" +"${SUDO[@]}" chmod 600 "$home_dir/.ssh/authorized_keys" +"${SUDO[@]}" chown "$remote_user" "$home_dir/.ssh/authorized_keys" 2>/dev/null || true + +ssh_config_path="$home_dir/.ssh/config" +ssh_config_tmp="$(mktemp /tmp/ssh_config.XXXXXX)" +ssh_config_new="${ssh_config_tmp}.new" + +if [[ -f "$ssh_config_path" ]]; then + "${SUDO[@]}" cp "$ssh_config_path" "$ssh_config_tmp" +else + : > "$ssh_config_tmp" +fi + +awk -v begin="$managed_begin" -v end="$managed_end" ' + $0 == begin { skip = 1; next } + $0 == end { skip = 0; next } + !skip { print } +' "$ssh_config_tmp" > "$ssh_config_new" + +printf '\n' >> "$ssh_config_new" +cat "$remote_ssh_config_file" >> "$ssh_config_new" + +if [[ "$(id -u)" -eq 0 ]]; then + cat "$ssh_config_new" > "$ssh_config_path" +else + "${SUDO[@]}" cp "$ssh_config_new" "$ssh_config_path" +fi +"${SUDO[@]}" chmod 600 "$ssh_config_path" +"${SUDO[@]}" chown "$remote_user" "$ssh_config_path" 2>/dev/null || true + +hosts_tmp="$(mktemp /tmp/etc_hosts.XXXXXX)" +hosts_new="${hosts_tmp}.new" + +if [[ -f /etc/hosts ]]; then + "${SUDO[@]}" cp /etc/hosts "$hosts_tmp" +else + : > "$hosts_tmp" +fi + +awk -v begin="$managed_begin" -v end="$managed_end" ' + $0 == begin { skip = 1; next } + $0 == end { skip = 0; next } + !skip { print } +' "$hosts_tmp" > "$hosts_new" + +printf '\n' >> "$hosts_new" +cat "$remote_hosts_file" >> "$hosts_new" + +if [[ "$(id -u)" -eq 0 ]]; then + cat "$hosts_new" > /etc/hosts +else + "${SUDO[@]}" cp "$hosts_new" /etc/hosts +fi +"${SUDO[@]}" chmod 644 /etc/hosts 2>/dev/null || true + +if command -v ssh-keyscan >/dev/null 2>&1; then + known_tmp="$(mktemp /tmp/known_hosts.XXXXXX)" + awk ' + $1 == "Host" { host = $2; ip = ""; port = "" } + $1 == "HostName" { ip = $2 } + $1 == "Port" { + port = $2 + if (host != "" && ip != "" && port != "") { + print ip, host, port + } + } + ' "$remote_ssh_config_file" | + while read -r ip host port; do + ssh-keyscan -T 5 -p "$port" "$host" "$ip" >> "$known_tmp" 2>/dev/null || true + done + + if [[ -s "$known_tmp" ]]; then + "${SUDO[@]}" touch "$home_dir/.ssh/known_hosts" + "${SUDO[@]}" chmod 644 "$home_dir/.ssh/known_hosts" + known_merged="$(mktemp /tmp/known_hosts_merged.XXXXXX)" + "${SUDO[@]}" cat "$home_dir/.ssh/known_hosts" "$known_tmp" | sort -u > "$known_merged" + if [[ "$(id -u)" -eq 0 ]]; then + cat "$known_merged" > "$home_dir/.ssh/known_hosts" + else + "${SUDO[@]}" cp "$known_merged" "$home_dir/.ssh/known_hosts" + fi + "${SUDO[@]}" chmod 644 "$home_dir/.ssh/known_hosts" + "${SUDO[@]}" chown "$remote_user" "$home_dir/.ssh/known_hosts" 2>/dev/null || true + rm -f "$known_merged" + fi + rm -f "$known_tmp" +else + echo "ssh-keyscan not found on $(hostname), skipped known_hosts update" >&2 +fi + +rm -f "$remote_hosts_file" "$remote_pubkeys_file" "$remote_ssh_config_file" "$hosts_tmp" "$hosts_new" "$ssh_config_tmp" "$ssh_config_new" +REMOTE +} + +verify_remote_login() { + local target="$1" + local source_hostname="$2" + local source_user="$3" + local host + shift 3 + + if ! run_ssh "$target" 'bash -s' -- "$source_hostname" "$source_user" "$@" <<'REMOTE' +set -euo pipefail + +source_hostname="$1" +source_user="$2" +shift 2 + +if [[ "$(id -un)" == "$source_user" ]]; then + RUN_AS=() +elif command -v sudo >/dev/null 2>&1 && sudo -n -u "$source_user" true 2>/dev/null; then + RUN_AS=(sudo -n -u "$source_user") +elif [[ "$(id -u)" -eq 0 && -x "$(command -v runuser 2>/dev/null)" ]]; then + RUN_AS=(runuser -u "$source_user" --) +else + for host in "$@"; do + [[ "$host" != "$source_hostname" ]] || continue + printf '%s -> %s (cannot run ssh as user %s)\n' "$source_hostname" "$host" "$source_user" + done + exit 0 +fi + +for host in "$@"; do + [[ "$host" != "$source_hostname" ]] || continue + if "${RUN_AS[@]}" ssh \ + -o BatchMode=yes \ + -o PasswordAuthentication=no \ + -o ConnectTimeout=8 \ + -o StrictHostKeyChecking=yes \ + "$host" true 2>/dev/null; then + echo "OK: $source_hostname -> $host" >&2 + else + printf '%s -> %s\n' "$source_hostname" "$host" + fi +done + +exit 0 +REMOTE + then + for host in "$@"; do + [[ "$host" != "$source_hostname" ]] || continue + printf '%s -> %s (could not run verification on source host)\n' "$source_hostname" "$host" + done + fi +} + +main() { + validate_config + check_local_connectivity + + local server_count="${#SSH_TARGETS[@]}" + + WORKDIR="$(mktemp -d /tmp/setup_cluster_ssh.XXXXXX)" + trap cleanup EXIT + + local hosts_file="$WORKDIR/hosts_block" + local ssh_config_file="$WORKDIR/ssh_config_block" + local pubkeys_file="$WORKDIR/public_keys" + local -a hostnames=() + local -a remote_login_failures=() + local i target hostname user pubkey failure + + { + printf '%s\n' "$MANAGED_BEGIN" + for ((i = 0; i < server_count; i++)); do + printf '%s %s\n' "${HOSTS_IPS[$i]}" "${HOST_NAMES[$i]}" + done + printf '%s\n' "$MANAGED_END" + } > "$hosts_file" + + { + printf '%s\n' "$MANAGED_BEGIN" + for ((i = 0; i < server_count; i++)); do + printf 'Host %s\n' "${HOST_NAMES[$i]}" + printf ' HostName %s\n' "${HOSTS_IPS[$i]}" + printf ' User %s\n' "${HOST_USERS[$i]}" + printf ' Port %s\n' "${HOST_PORTS[$i]}" + printf '\n' + done + printf '%s\n' "$MANAGED_END" + } > "$ssh_config_file" + + for ((i = 0; i < server_count; i++)); do + hostnames+=("${HOST_NAMES[$i]}") + done + + : > "$pubkeys_file" + + log "checking/generating remote SSH keys" + for ((i = 0; i < server_count; i++)); do + target="${SSH_TARGETS[$i]}" + hostname="${HOST_NAMES[$i]}" + user="${HOST_USERS[$i]}" + log "key check: $target ($hostname, user $user)" + pubkey="$(ensure_remote_key_and_print_pubkey "$target" "$user" | awk '/^ssh-(ed25519|rsa) / { key = $0 } END { if (key) print key }')" + [[ -n "$pubkey" ]] || die "Could not read public key for user $user from $target" + printf '%s\n' "$pubkey" >> "$pubkeys_file" + done + + sort -u "$pubkeys_file" -o "$pubkeys_file" + + log "updating /etc/hosts, authorized_keys, and known_hosts on every server" + for ((i = 0; i < server_count; i++)); do + target="${SSH_TARGETS[$i]}" + hostname="${HOST_NAMES[$i]}" + user="${HOST_USERS[$i]}" + log "install: $target ($hostname, user $user)" + install_hosts_keys_and_known_hosts "$target" "$hosts_file" "$pubkeys_file" "$ssh_config_file" "$user" + done + + if [[ "$VERIFY_REMOTE_LOGIN" == "true" ]]; then + log "verifying remote-to-remote SSH login" + for ((i = 0; i < server_count; i++)); do + target="${SSH_TARGETS[$i]}" + hostname="${HOST_NAMES[$i]}" + user="${HOST_USERS[$i]}" + log "verify from: $hostname as $user" + while IFS= read -r failure; do + [[ -n "$failure" ]] || continue + remote_login_failures+=("$failure") + done < <(verify_remote_login "$target" "$hostname" "$user" "${hostnames[@]}") + done + + if [[ "${#remote_login_failures[@]}" -gt 0 ]]; then + printf '\nRemote passwordless SSH failed for these hostname pairs:\n' >&2 + for failure in "${remote_login_failures[@]}"; do + printf ' - %s\n' "$failure" >&2 + done + exit 1 + fi + + log "remote-to-remote SSH verification passed" + fi + + log "done" +} + +parse_args "$@" +main \ No newline at end of file diff --git a/agents/skills/README.md b/agents/skills/README.md new file mode 100644 index 000000000..73a4591f0 --- /dev/null +++ b/agents/skills/README.md @@ -0,0 +1,5 @@ +## Shared Skills + +This directory contains skills that can be reused by multiple agents. + +Agent-specific skills should live under `agents//skills/` instead of this shared directory. diff --git a/agents/uninstall.sh b/agents/uninstall.sh new file mode 100755 index 000000000..7b8b8de80 --- /dev/null +++ b/agents/uninstall.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +usage() { + cat < --tool --scope +EOF +} + +fail() { + echo "Error: $*" >&2 + exit 1 +} + +parse_args() { + AGENT="" + TOOL="" + SCOPE="" + + while [ "$#" -gt 0 ]; do + case "$1" in + --agent) + [ "$#" -ge 2 ] || fail "--agent requires a value" + AGENT="$2" + shift 2 + ;; + --tool) + [ "$#" -ge 2 ] || fail "--tool requires a value" + TOOL="$2" + shift 2 + ;; + --scope) + [ "$#" -ge 2 ] || fail "--scope requires a value" + SCOPE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac + done + + [ -n "$AGENT" ] || fail "missing --agent " + [ -n "$TOOL" ] || fail "missing --tool " + [ -n "$SCOPE" ] || fail "missing --scope " + + case "$TOOL" in + claude|codex) ;; + *) fail "--tool must be claude or codex" ;; + esac + + case "$SCOPE" in + project|global) ;; + *) fail "--scope must be project or global" ;; + esac +} + +project_root() { + cd "$ROOT_DIR/.." && pwd +} + +claude_target() { + if [ "$SCOPE" = "project" ]; then + printf '%s/.claude/agents/%s.md\n' "$(project_root)" "$AGENT" + else + printf '%s/.claude/agents/%s.md\n' "$HOME" "$AGENT" + fi +} + +codex_target() { + if [ "$SCOPE" = "project" ]; then + printf '%s/AGENTS.md\n' "$(project_root)" + else + printf '%s/.codex/AGENTS.md\n' "$HOME" + fi +} + +parse_args "$@" + +uninstall_claude() { + local target_file + target_file="$(claude_target)" + + if [ -f "$target_file" ]; then + rm -f "$target_file" + echo "Removed Claude agent '$AGENT' from $target_file" + else + echo "Claude target not found: $target_file" + fi +} + +uninstall_codex() { + local target_file + target_file="$(codex_target)" + + if [ ! -f "$target_file" ]; then + echo "Codex target not found: $target_file" + return 0 + fi + + python3 - "$target_file" "$AGENT" <<'PY' +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +agent = sys.argv[2] +begin = f"" +end = f"" + +text = path.read_text(encoding="utf-8") +start = text.find(begin) + +if start < 0: + print(f"Codex block not found for agent '{agent}' in {path}") + raise SystemExit(0) + +stop = text.find(end, start) +if stop < 0: + print(f"Error: found {begin} without matching {end} in {path}", file=sys.stderr) + raise SystemExit(1) + +stop += len(end) +new_text = text[:start].rstrip() + "\n" + text[stop:].lstrip("\n") +path.write_text(new_text.rstrip() + ("\n" if new_text.strip() else ""), encoding="utf-8") +print(f"Removed Codex block for agent '{agent}' from {path}") +PY +} + +case "$TOOL" in + claude) + uninstall_claude + ;; + codex) + uninstall_codex + ;; +esac From 991f2fc0f9f2907a5757e4e10ff49ae1709f7e28 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Fri, 26 Jun 2026 16:36:42 +0000 Subject: [PATCH 2/6] feat(agents): harden pixels install workflow --- agents/README.md | 1 + agents/pixels-install/agent.yaml | 12 +- agents/pixels-install/claude.md | 135 ++++-- agents/pixels-install/codex.md | 135 ++++-- .../scripts/build_install_pixels.sh | 72 ++- .../scripts/check_prerequisites.sh | 129 ++++-- .../scripts/configure_pixels.sh | 74 ++- agents/pixels-install/scripts/install_etcd.sh | 272 ++++++++++- agents/pixels-install/scripts/install_jdk.sh | 199 ++++++++ .../pixels-install/scripts/install_maven.sh | 78 ++-- .../pixels-install/scripts/install_mysql.sh | 266 +++++++++++ .../scripts/install_shell_helpers.sh | 131 ++++++ .../pixels-install/scripts/install_trino.sh | 387 ++++++++++++++++ .../scripts/install_trino_cluster.sh | 184 ++++++++ .../scripts/install_trino_shell_helpers.sh | 218 +++++++++ .../pixels-install/scripts/lib/shell_env.sh | 323 +++++++++++++ .../scripts/prepare_deployment.sh | 217 ++++++++- .../scripts/prepare_trino_cluster.sh | 429 ++++++++++++++++++ agents/pixels-install/scripts/progress.sh | 86 ++++ agents/pixels-install/scripts/smoke_test.sh | 242 +++++++--- 20 files changed, 3366 insertions(+), 224 deletions(-) create mode 100755 agents/pixels-install/scripts/install_jdk.sh create mode 100755 agents/pixels-install/scripts/install_mysql.sh create mode 100755 agents/pixels-install/scripts/install_shell_helpers.sh create mode 100755 agents/pixels-install/scripts/install_trino.sh create mode 100755 agents/pixels-install/scripts/install_trino_cluster.sh create mode 100755 agents/pixels-install/scripts/install_trino_shell_helpers.sh create mode 100644 agents/pixels-install/scripts/lib/shell_env.sh create mode 100755 agents/pixels-install/scripts/prepare_trino_cluster.sh create mode 100755 agents/pixels-install/scripts/progress.sh diff --git a/agents/README.md b/agents/README.md index 460c5bdd1..947fffb09 100644 --- a/agents/README.md +++ b/agents/README.md @@ -13,6 +13,7 @@ - `agents/scripts/`: Shared runtime scripts that can be reused by multiple agents. - `agents//skills/`: Private skills used only by that agent. - `agents//scripts/`: Private runtime scripts used only by that agent. +- `agents//scripts/lib/`: Shared shell functions `source`d by that agent's own scripts (not meant to be executed directly, so files here are not marked executable). - `agents/scripts/`: Helper scripts used internally by the installer. ### Commands diff --git a/agents/pixels-install/agent.yaml b/agents/pixels-install/agent.yaml index 51f1e1a8f..4050f818f 100644 --- a/agents/pixels-install/agent.yaml +++ b/agents/pixels-install/agent.yaml @@ -7,12 +7,20 @@ entrypoints: scripts: prepare: scripts/prepare_deployment.sh check: scripts/check_prerequisites.sh + jdk: scripts/install_jdk.sh maven: scripts/install_maven.sh + mysql: scripts/install_mysql.sh etcd: scripts/install_etcd.sh pixels: scripts/build_install_pixels.sh configure: scripts/configure_pixels.sh + shell_helpers: scripts/install_shell_helpers.sh smoke_test: scripts/smoke_test.sh + progress: scripts/progress.sh cluster_ssh: ../scripts/setup_cluster.sh + trino_prepare: scripts/prepare_trino_cluster.sh + trino_install: scripts/install_trino.sh + trino_install_cluster: scripts/install_trino_cluster.sh + trino_shell_helpers: scripts/install_trino_shell_helpers.sh phases: - discover_install_target - check_prerequisites @@ -24,5 +32,7 @@ phases: - build_install_pixels - configure_pixels - start_pixels_optional + - install_shell_helpers_optional - smoke_test - - optional_trino_or_hadoop_or_monitoring + - optional_trino_install + - optional_hadoop_or_monitoring diff --git a/agents/pixels-install/claude.md b/agents/pixels-install/claude.md index 9c2728f24..aabd94f88 100644 --- a/agents/pixels-install/claude.md +++ b/agents/pixels-install/claude.md @@ -13,59 +13,132 @@ Treat `docs/INSTALL.md` as the source of truth for the installation flow. Use sc Use these helper scripts when they directly fit the current environment and the user-approved goal: -- `prepare_deployment.sh`: write `deployment.env` from cluster node input and optionally delegate SSH setup to `agents/scripts/setup_cluster.sh`. -- `check_prerequisites.sh`: validate OS, architecture, memory, disk, ports, host resolution, privilege, and optional SSH reachability. -- `install_maven.sh`: install or validate Maven 3.8+ when the current Maven is missing or incompatible with the selected JDK. -- `install_etcd.sh`: install and optionally start the bundled etcd 3.3.4 package. -- `build_install_pixels.sh`: build and install Pixels into `PIXELS_HOME`, then add the MySQL JDBC connector. -- `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, and path values are known. -- `smoke_test.sh`: verify the installed layout, configuration, metadata access, etcd health, and basic CLI behavior when applicable. +- `prepare_deployment.sh`: collect and confirm the Pixels deployment topology before any install writes to disk. With no node arguments it prompts for single-node vs. cluster, `PIXELS_HOME`, the coordinator node, whether the coordinator also runs a worker, and worker nodes; with arguments, use `--coordinator ` plus repeated `--worker <...>` (legacy `--node` still works). Writes `deployment.env` and optionally delegates SSH setup to `agents/scripts/setup_cluster.sh`. In non-interactive mode, set `CONFIRM_PIXELS_DEPLOYMENT=true` only after the user has reviewed the chosen topology and paths. +- `check_prerequisites.sh`: validate OS, architecture, memory, disk, ports, host resolution, privilege, and optional SSH reachability. Runs every check and exits with a structured `=== check_prerequisites result ===` summary (one `ok|warn|fail|skip : ` line per check, plus a final `summary: ok=N warn=N fail=N skip=N status=pass|fail` line) instead of stopping at the first failure, so a single run shows every problem at once. +- `install_jdk.sh`: install a Zulu OpenJDK build (default JDK 23) matching the server's CPU architecture (x86_64 or aarch64) under `~/opt`, and persist `JAVA_HOME` only into the current user's shell profile. Looks for a JDK that already satisfies `JDK_VERSION` first — checking `JAVA_HOME` if set, otherwise resolving whatever `java` is on `PATH` (e.g. an `apt`-installed JDK that never exported `JAVA_HOME`) — and if one is found, skips the download entirely and only fixes up the environment variable to point at it. Only downloads/installs a fresh Zulu build when nothing on the system satisfies the version requirement. Falls back to pointing at the manual `.deb` method in `docs/INSTALL.md` if the Azul metadata API lookup fails. +- `install_maven.sh`: install or validate Maven 3.8+ (default pinned version `3.9.8`; override with `MAVEN_VERSION` only if the user explicitly asks for a different one) when the current Maven is missing or incompatible with the selected JDK. Same "skip if already satisfied" logic as `install_jdk.sh`: if an existing `mvn` (from `apt`, a prior manual install, etc.) already meets the minimum version, it reuses that installation's home directory instead of downloading anything. Fresh installs go into `~/opt/apache-maven-` with a `~/opt/maven` symlink, never into `/opt`. +- `install_mysql.sh`: install MySQL, then interactively confirm the root password and the `pixels` metadata-DB-user password (default `password` for both, never applied silently), create the `pixels_metadata` database/user, and load `scripts/sql/metadata_schema.sql`. It validates DB/user identifiers, quotes SQL values, avoids putting the root password in command-line arguments, and writes shell-quoted `agents/pixels-install/deployment.secrets.env` (mode 600, untracked) so the confirmed credentials flow into `configure_pixels.sh` automatically. +- `install_etcd.sh`: install and optionally start the bundled etcd 3.3.4 package under `~/opt`. Always fixes the shipped `conf.yml`'s hardcoded `/home/ubuntu/...` `data-dir` to the real install path. By default (`ETCD_ALLOW_REMOTE=false`) it keeps etcd localhost-only. For a cluster, set `ETCD_ALLOW_REMOTE=true` only after confirming private networking/security-group rules; the script then requires `CONFIRM_ETCD_REMOTE_ACCESS=true`, `ASSUME_YES=true`, or an interactive confirmation before binding the client/peer listeners for remote access. Can also install and enable a `systemd` unit so etcd survives reboots and restarts on failure, but only after asking — it never installs this silently. Leave `INSTALL_ETCD_SYSTEMD_SERVICE` unset to be prompted interactively (`[y/N]`, defaults to no); set it to `true`/`false` to answer non-interactively once the user has actually agreed, or `ASSUME_YES=true` to answer yes to this and other yes/no prompts. Declining (or running non-interactively with no explicit answer) just skips the unit — etcd still starts, as a manually backgrounded process instead. Falls back the same way when `systemctl` isn't available at all. Restarts/re-enables the installed service automatically when the config actually changed. +- `build_install_pixels.sh`: build and install Pixels into the confirmed `PIXELS_HOME` from `deployment.env`, then add the MySQL JDBC connector. It prints the repo, deployment file, and install target before running; in non-interactive mode, set `CONFIRM_PIXELS_INSTALL=true` only after the user confirms the target. If you must bypass the prepared deployment file, pass `USE_DEPLOYMENT_FILE=false PIXELS_HOME=` explicitly. `AUTO_CONFIRM_INSTALL` (default `true`) auto-answers `install.sh`'s prompts only when that is safe; if `PIXELS_HOME/etc/pixels.properties` or `pixels-cpp.properties` already exist, it forces the interactive path instead, because `install.sh` would otherwise prompt to add/remove config options or overwrite `pixels-cpp.properties` outright — exactly what `CLAUDE.md`/`AGENTS.md` say not to do without the user's say-so. +- `configure_pixels.sh`: update `PIXELS_HOME/etc/pixels.properties` after database, etcd, host, port, topology, and path values are known. Automatically sources `deployment.env` and `deployment.secrets.env` (if present), so `PIXELS_HOME`, coordinator service hosts, worker names, and MySQL credentials stay in sync with what the earlier scripts confirmed. It writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS` by default. It fails instead of silently writing the documented default DB password unless `METADATA_DB_PASSWORD` is explicit or `ALLOW_DEFAULT_METADATA_PASSWORD=true` is set. +- `install_shell_helpers.sh`: optional convenience step. Writes `start_pixels`/`stop_pixels`/`restart_pixels` shell functions to `~/.pixels-shell-helpers.sh` and sources that file from the current user's shell profile. These wrap `$PIXELS_HOME/sbin/{start,stop}-pixels.sh`; they only also drive Trino (`$TRINO_HOME/bin/launcher`) if the user's shell already has `TRINO_HOME` set, and skip that step (without failing) otherwise. Purely user-scoped (no system-wide change), so unlike the etcd systemd unit it does not need a confirmation prompt. +- `smoke_test.sh`: verify the installed layout, configuration, metadata access, etcd health, Pixels topology (`deployment.env` plus `$PIXELS_HOME/etc/workers`), and basic CLI behavior when applicable. It also checks Trino layout/catalog files when `CHECK_TRINO=true` and `trino-deployment.env` is present. Same structured-summary behavior as `check_prerequisites.sh` above. +- `progress.sh`: records which of agent.yaml's `phases` have actually completed, in `agents/pixels-install/progress.env` (generated, untracked). `progress.sh mark [note]` after a phase succeeds, `progress.sh show` to see what's already done, `progress.sh is-done ` to check one phase, `progress.sh reset` to start over. Exists so a session interrupted partway through a multi-phase install can resume from the actual state instead of guessing or re-running completed steps. +- `install_trino_cluster.sh`: run **on the coordinator** after `prepare_trino_cluster.sh`. It prints and confirms the Trino deployment file, version, install parent, home link, data dir, coordinator, coordinator scheduling mode, and workers before installing; in non-interactive mode, set `CONFIRM_TRINO_CLUSTER_INSTALL=true` only after the user confirms the fan-out. Installs Trino locally for the coordinator's own role, then dispatches `install_trino.sh` on every worker **in parallel** over the coordinator → worker passwordless SSH that `prepare_trino_cluster.sh` already set up, instead of requiring the user (or the agent) to copy the same command and run it by hand once per node. Copies the current `trino-deployment.env` to each worker first (workers don't need a shared filesystem), and assumes the repository already exists at the same path on every worker (override `REMOTE_REPO_ROOT` otherwise — the same assumption `install_trino.sh` itself already documents). Emits the same structured per-node summary as `smoke_test.sh`/`check_prerequisites.sh`, with a log file per node under `agents/pixels-install/logs/` for diagnosing any failure. +- `prepare_trino_cluster.sh`: collects and confirms the Trino topology and install locations: one coordinator, zero or more workers, whether the coordinator also accepts scheduled work, Trino version, HTTP port, install parent, home symlink/current path, and data directory. With no node arguments it prompts interactively; with arguments, use `--coordinator ` plus repeated `--worker <...>`. Writes `agents/pixels-install/trino-deployment.env`. Optionally delegates to `agents/scripts/setup_cluster.sh` (`--setup-ssh true`) so the coordinator and every worker can passwordlessly SSH into each other, which the cluster shell helpers below depend on. In non-interactive mode, set `CONFIRM_TRINO_DEPLOYMENT=true` only after the user has reviewed the topology and paths. +- `install_trino.sh`: installs Trino (per the [466 deployment docs](https://trino.io/docs/466/installation/deployment.html)) on the **current node only** — run it once per node in the cluster, it does not SSH anywhere itself. It reads `trino-deployment.env`, validates this node's role (`coordinator` or `worker`) and the coordinator host, and confirms the role plus install/data paths before writing files; in non-interactive mode, set `CONFIRM_TRINO_INSTALL=true` only after the user confirms the target. Downloads `trino-server-.tar.gz` (default 466) from Maven Central into the confirmed install parent with the confirmed home symlink (`change_trino_version`-friendly). Writes `etc/node.properties` (`node.data-dir` under the confirmed data dir, outside the versioned install dir so it survives version switches; `node.id` generated once and preserved across re-runs), `etc/jvm.config` (official defaults plus the two `--add-opens` flags `pixels-trino` requires), `etc/log.properties`, and `etc/config.properties` — `discovery.uri` always points at the coordinator's real IP from `trino-deployment.env`, never `localhost`; this node's coordinator/worker role is read from `trino-deployment.env` or inferred from this host's own IPs, overridable with `TRINO_ROLE`. Also clones/builds `pixelsdb/pixels-trino` (requires Pixels already `mvn install`ed locally and JDK 23) and installs the connector into `plugin/`, writing `etc/catalog/pixels.properties` (`cloud.function.switch=off` by default — Pixels Turbo stays out of scope unless asked). The event listener plugin is optional and off by default (`INSTALL_PIXELS_TRINO_LISTENER=true` to enable). Best-effort installs `trino-cli` into `bin/trino`. +- `install_trino_shell_helpers.sh`: optional, and — unlike `install_shell_helpers.sh` — **always asks first**, because it bakes a specific list of remote hostnames (from `trino-deployment.env`) into the user's shell profile and assumes passwordless SSH between them is already set up. Writes `start_trino_cluster`/`stop_trino_cluster`/`restart_trino_cluster`/`change_trino_version`/`trino_cli` to `~/.trino-shell-helpers.sh`: the coordinator is always operated locally, every worker over SSH. Leave `INSTALL_TRINO_SHELL_HELPERS` unset to be prompted `[y/N]`; set `true`/`false` to answer non-interactively once the user has actually agreed, or `ASSUME_YES=true`. + +All of these scripts anchor paths on `$HOME` (e.g. `~/opt/...`) and detect the +current login shell (`$SHELL`) to pick `~/.bashrc` or `~/.zshrc` before +persisting any `export`. Never assume the deployment user is named `pixels` +or `ubuntu`, and never write environment variables into a global file such +as `/etc/environment` — `agents/pixels-install/scripts/lib/shell_env.sh` +centralizes this logic and is shared by `install_jdk.sh`, `install_maven.sh`, +`install_etcd.sh`, `build_install_pixels.sh`, `configure_pixels.sh`, +`check_prerequisites.sh`, `smoke_test.sh`, `install_trino.sh`, +`install_trino_cluster.sh`, `prepare_trino_cluster.sh`, +`install_trino_shell_helpers.sh`, and `progress.sh`. It also defines the +structured-result (`result_record`/`result_emit_summary`) and phase-checkpoint +(`mark_phase_done`/`phase_is_done`/`print_progress`) helpers described above. + +Each `install_*.sh` script runs as its own process, so exporting a variable +in one script does not make it visible to the next one. `docs/INSTALL.md`'s +own walkthrough handles this by running `source ~/.bashrc` right after +appending to it, but sourcing the user's real rc file from a *non*-interactive +script is unreliable (Ubuntu's default `~/.bashrc` returns immediately for +non-interactive shells, before reaching anything appended at the bottom). So +in addition to persisting into the user's real shell profile, +`shell_env.sh` also mirrors `JAVA_HOME`/`MAVEN_HOME`/`ETCD`/`PIXELS_HOME` +into a small `agents/pixels-install/toolchain.env` file with no such guard, +and every script sources it (`load_toolchain_env`) at startup. This means +running `install_jdk.sh` then `install_maven.sh` as two separate tool calls +still works without the agent needing to manually re-source anything in +between — but if you set any of these variables yourself before running a +step, your explicit environment variable still wins. The agent may run simple commands from `docs/INSTALL.md` directly when that is clearer than adding another wrapper script. ## Workflow -1. Clarify the installation target: single node or cluster, `PIXELS_HOME`, storage/query engine needs, whether to start services now, and whether optional components are in scope. -2. Inspect the current environment before installing anything: Java, Maven, MySQL, etcd, ports, disk, memory, repository location, and existing `PIXELS_HOME`. -3. Prepare cluster configuration only when needed. Prefer repeated `--node ssh_target,host_ip,host_name,host_user,host_port` arguments or `--nodes-file `. -4. Install or verify JDK according to `docs/INSTALL.md`. JDK 23+ is required for the documented Pixels + Trino 466 path; other query engines may allow different versions. Ask before installing a downloaded JDK package. -5. Install or verify Maven 3.8+ and ensure `mvn -v` uses the selected JDK. -6. Install Pixels from the repository and place the MySQL JDBC connector under `PIXELS_HOME/lib`. -7. Configure MySQL metadata storage only after the user confirms credentials, root access method, and whether remote access should be enabled. Do not run `mysql_secure_installation` non-interactively. -8. Install or verify etcd, then confirm endpoint health. -9. Update `pixels.properties` with the confirmed paths, hosts, ports, database settings, etcd settings, cache setting, and optional Trino JDBC URL. -10. Start Pixels only when the user asks to start services or when startup is part of the requested installation task. -11. Run smoke checks after configuration or startup changes. +The phase sequence is fixed in `agent.yaml`'s `phases` list — this section is decision guidance for each phase, not a second copy of that ordering. Before doing anything else, run `scripts/progress.sh show` to see which phases (if any) already completed in a prior session, and skip straight to the first phase that is not yet marked done instead of re-deriving state from scratch or blindly redoing finished work. After each phase actually succeeds, run `scripts/progress.sh mark ` (phase names match `agent.yaml`) so a later session — or this one, after a long gap — can pick up correctly. + +- **discover_install_target**: clarify single node vs. cluster, `PIXELS_HOME`, coordinator vs. worker roles, storage/query engine needs, whether to start services now, and whether optional components are in scope. Determine the actual login user (`whoami`) and home directory (`$HOME`) instead of assuming `pixels` or `ubuntu`; every path this agent writes is anchored on that `$HOME`. Do not proceed with defaults until the user has confirmed the topology and install locations. +- **check_prerequisites**: run `check_prerequisites.sh` and read its full structured summary before deciding what to install — see "Failure Handling" below for what to do with anything it reports as `fail`. +- **prepare_deployment_config**: run for both single-node and cluster deployments so later scripts read the same confirmed `PIXELS_HOME`, coordinator service hosts, and worker list from `deployment.env`. Prefer `prepare_deployment.sh` with no node arguments for an interactive prompt, or explicit `--coordinator ... --worker ...` arguments when the user has already provided the topology. +- **install_or_verify_jdk**: `install_jdk.sh` already skips the download when anything on the system (apt-installed JDK, a previous run, etc.) satisfies `JDK_VERSION`; it only selects and installs a Zulu build under `~/opt` when nothing qualifies. JDK 23+ is required for the documented Pixels + Trino 466 path. Ask before installing a downloaded JDK package. +- **install_or_verify_maven**: `install_maven.sh` has the same "skip if already satisfied" behavior for Maven 3.8+ (default pinned `3.9.8`, only overridden if the user asks). Ensure `mvn -v` ends up using the selected JDK. +- **install_or_verify_mysql** / **install_or_verify_etcd**: only after the user confirms credentials/ports (see Guardrails — never apply the default MySQL password silently); `install_etcd.sh` keeps localhost-only by default, asks before remote etcd access, and asks before installing its `systemd` auto-start unit. +- **build_install_pixels**: install Pixels from the repository into the confirmed `PIXELS_HOME` from `deployment.env` and place the MySQL JDBC connector under `PIXELS_HOME/lib`. +- **configure_pixels**: update `pixels.properties` only once paths, hosts, ports, database, etcd, topology, and cache settings are confirmed; it derives `pixels.var.dir` from the resolved `PIXELS_HOME`, writes `$PIXELS_HOME/etc/workers` from `PIXELS_WORKERS`, and reuses the MySQL credentials `install_mysql.sh` already confirmed. If MySQL was configured elsewhere, pass `METADATA_DB_PASSWORD` explicitly. +- **start_pixels_optional**: only when the user asks to start services or startup is part of the requested task. +- **install_shell_helpers_optional**: only if the user wants `start_pixels`/`stop_pixels`/`restart_pixels` convenience functions. +- **smoke_test**: run after configuration or startup changes; read the full structured summary (see "Failure Handling" below), don't just check the exit code. Use `CHECK_TRINO=true` only when Trino was installed or explicitly requested. +- **optional_trino_install**: only when the user explicitly asks. Clarify the topology first: coordinator vs. worker(s), how many workers, whether the coordinator also accepts scheduled work (`node-scheduler.include-coordinator`; official docs recommend `false` for a dedicated coordinator, `true` for a single-node test setup), `TRINO_INSTALL_PARENT`, `TRINO_HOME_LINK`, and `TRINO_DATA_DIR`. Run `prepare_trino_cluster.sh --setup-ssh true`, then run `install_trino_cluster.sh` **on the coordinator** to install Trino on the coordinator and fan out to every worker in parallel over SSH, instead of repeating `install_trino.sh` by hand once per node. Configure the catalog and `presto.pixels.jdbc.url` only once the coordinator's endpoint and catalog/schema are known. Offer `install_trino_shell_helpers.sh` only after the user agrees (it asks by default anyway). +- **optional_hadoop_or_monitoring**: out of scope unless the user asks — see Optional Components below. + +## Failure Handling + +`check_prerequisites.sh` and `smoke_test.sh` never stop at the first failed check — they run every check and print a structured summary block (`===