From 0b272374582b1bcc63083745bd6d28faf07201e9 Mon Sep 17 00:00:00 2001 From: roby Date: Wed, 9 Sep 2026 09:43:31 -0600 Subject: [PATCH 1/6] Firefly-2099: Harden and improve Firefly standalone install - add to build_publis an option make only standalone - get_firefly now has a development mode to pull from a branch build - more checking after each download - java install less fragile - documenation has better trouble shooting - fallback clean up of redis - fix a couple of bug --- .github/workflows/build_publish.yml | 37 ++++- bin/get-firefly | 88 ++++++++++-- bin/install.sh | 136 +++++++++++++++--- docs/using-firefly-standalone.md | 17 ++- src/firefly/js/ui/FileUpload.jsx | 6 +- src/standalone/assets/ff | 52 ++++++- src/standalone/assets/javaInstaller.sh | 46 +++++- src/standalone/assets/jreVersion.json | 20 +-- src/standalone/assets/startFireflyServer.sh | 19 ++- .../caltech/ipac/app/FireflyApplication.java | 7 +- 10 files changed, 373 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build_publish.yml b/.github/workflows/build_publish.yml index abdb1d4101..09035d9587 100644 --- a/.github/workflows/build_publish.yml +++ b/.github/workflows/build_publish.yml @@ -4,11 +4,11 @@ on: workflow_dispatch: inputs: git_tag: - description: "Git tag to use(e.g. 'release-xxxx.x.x' or a full git tag)" + description: "Git tag or branch to use (e.g. 'release-xxxx.x.x' or a full git tag)" required: true type: string img_tag: - description: "Optional: Docker image tag(e.g. 'xxxx.x.x'); If omitted, `git_tag` will be used." + description: "Optional: Docker image tag (e.g. 'xxxx.x.x'); If omitted, `git_tag` will be used." required: false type: string push_image: @@ -20,6 +20,11 @@ on: description: "Optional: Firefly release tag for standalone.zip artifact (e.g. release-xxxx.x.x)" required: false type: string + standalone_only: + description: "standalone.zip only (no docker push, no GitHub release)" + required: false + default: false + type: boolean release: types: [published] @@ -90,7 +95,7 @@ jobs: # Login to GHCR (only if pushing) # ------------------------------------------------------------ - name: Login to GHCR - if: github.event_name == 'release' || inputs.push_image + if: (github.event_name == 'release' || inputs.push_image) && !inputs.standalone_only uses: docker/login-action@v3 with: registry: ghcr.io @@ -106,7 +111,7 @@ jobs: context: . file: firefly/docker/Dockerfile platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name == 'release' || inputs.push_image }} + push: ${{ (github.event_name == 'release' || inputs.push_image) && !inputs.standalone_only}} # docker buildx does not allow uppercase letters in tags, so we convert to lowercase here tags: ghcr.io/caltech-ipac/firefly:${{ steps.resolve_tags.outputs.tag }} build-args: | @@ -120,7 +125,7 @@ jobs: # `standalone-builder` layer (gradle standalone:zip) runs. # ------------------------------------------------------------ - name: Build and export standalone.zip - if: github.event_name == 'release' || inputs.release_tag != '' + if: github.event_name == 'release' || inputs.release_tag != '' || inputs.standalone_only uses: docker/build-push-action@v6 with: context: . @@ -140,13 +145,31 @@ jobs: # using `gh release upload` requires setting `contents: write` # ------------------------------------------------------------ - name: Add standalone.zip to release artifacts - if: github.event_name == 'release' || inputs.release_tag != '' + if: (github.event_name == 'release' || inputs.release_tag != '') && !inputs.standalone_only env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | TAG="${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release_tag }}" gh release upload "$TAG" --repo "${{ github.repository }}" ./dist-zip/standalone.zip --clobber + # ------------------------------------------------------------ + # standalone_only: publish standalone.zip as a workflow + # artifact instead of a release asset. Ephemeral, private to the run. + # ------------------------------------------------------------ + - name: Upload standalone.zip as workflow artifact + id: upload_standalone_artifact + if: inputs.standalone_only + uses: actions/upload-artifact@v4 + with: + name: standalone-zip + path: ./dist-zip/standalone.zip + + - name: Print artifact URL + if: inputs.standalone_only + run: | + echo "Artifact URL: ${{ steps.upload_standalone_artifact.outputs.artifact-url }}" >> $GITHUB_STEP_SUMMARY + echo "nightly.link (no login required): https://nightly.link/${{ github.repository }}/actions/runs/${{ github.run_id }}/standalone-zip.zip" >> $GITHUB_STEP_SUMMARY + # ------------------------------------------------------------ # Package and push Helm chart to GHCR # ------------------------------------------------------------ @@ -154,7 +177,7 @@ jobs: uses: azure/setup-helm@v4 - name: Publish Helm chart - if: github.event_name == 'release' || inputs.push_image + if: (github.event_name == 'release' || inputs.push_image) && !inputs.standalone_only run: | if [[ ! -f firefly/helm/Chart.yaml ]]; then echo "No Helm chart found, skipping." diff --git a/bin/get-firefly b/bin/get-firefly index 78d60b402f..55cf2e3162 100644 --- a/bin/get-firefly +++ b/bin/get-firefly @@ -1,9 +1,81 @@ #!/bin/bash -INSTALL_SCRIPT="https://raw.githubusercontent.com/Caltech-IPAC/firefly/refs/heads/dev/bin/install.sh" -# use the following line for testing the code from a PR, modify for your PR -# INSTALL_SCRIPT="https://raw.githubusercontent.com/Caltech-IPAC/firefly/refs/heads/FIREFLY-1980-standalone/bin/install.sh" - -curl -s ${INSTALL_SCRIPT} > ./install.sh -chmod +x ./install.sh -./install.sh -dontConfirm "$@" -/bin/rm -f ./install.sh \ No newline at end of file + +# -------------------------- +# This script start the Firefly install on an end-user machine +# -------------------------- + +# -------------------------- +# How to use +# +# Production: +# curl -L https://raw.githubusercontent.com/Caltech-IPAC/firefly/refs/heads/dev/bin/get-firefly | bash +# +# Testing in a branch: +# curl -L https://raw.githubusercontent.com/Caltech-IPAC/firefly/refs/heads/FIREFLY-2099-stand-hard/bin/get-firefly | FIREFLY_BRANCH=Firefly-xxx bash +# -------------------------- + +ref="${FIREFLY_BRANCH:-dev}" +INSTALL_SCRIPT="https://raw.githubusercontent.com/Caltech-IPAC/firefly/refs/heads/${ref}/bin/install.sh" + +downloadedInstallScript="" +jobArtifactZip="" +jobExtractDir="" +trap '[ -n "$downloadedInstallScript" ] && rm -f "$downloadedInstallScript"; [ -n "$jobArtifactZip" ] && rm -f "$jobArtifactZip"; [ -n "$jobExtractDir" ] && rm -rf "$jobExtractDir"' EXIT + +downloadedInstallScript=$(mktemp -t firefly-install.XXXXXX) || { echo "Failed to create a temporary file"; exit 1; } + +curl -fsSL "${INSTALL_SCRIPT}" -o "$downloadedInstallScript" +curlStatus=$? +if [ $curlStatus -ne 0 ] || [ ! -s "$downloadedInstallScript" ]; then + echo "Failed to download the Firefly installer from $INSTALL_SCRIPT" + exit 1 +fi +chmod +x "$downloadedInstallScript" + +# -------------------------- +# The following two environment variables are only for testing. + +# FIREFLY_RUN_ID: when set, install the build of standalone.zip from that +# specific GitHub Actions run (via nightly.link) instead of the latest formal GitHub +# release. If testing is set up correctly then FIREFLY_RUN_ID should not be necessary +# +# FIREFLY_BRANCH: if FIREFLY_RUN_ID isn't set but this is, auto-resolve the latest +# successful workflow_dispatch run on that branch. +# Note this can't tell whether that run was actually a standalone_only +# build - if not, nightly.link will just report no artifact found for it. +# +# -------------------------- + +installUrlArgs=() +if [ -z "$FIREFLY_RUN_ID" ] && [ -n "$FIREFLY_BRANCH" ]; then + if ! command -v jq > /dev/null 2>&1; then + echo "jq is required to resolve FIREFLY_BRANCH to a run id; install jq or set FIREFLY_RUN_ID directly" + exit 1 + fi + runApiUrl="https://api.github.com/repos/Caltech-IPAC/firefly/actions/workflows/build_publish.yml/runs?branch=${FIREFLY_BRANCH}&status=success&event=workflow_dispatch" + FIREFLY_RUN_ID=$(curl -fsSL "$runApiUrl" | jq -r '.workflow_runs[0].id // empty') + if [ -z "$FIREFLY_RUN_ID" ]; then + echo "No successful workflow_dispatch run found for branch $FIREFLY_BRANCH" + exit 1 + fi +fi + +if [ -n "$FIREFLY_RUN_ID" ]; then + runLinkUrl="https://nightly.link/Caltech-IPAC/firefly/actions/runs/${FIREFLY_RUN_ID}/standalone-zip.zip" + jobArtifactZip=$(mktemp -t firefly-build-artifact.XXXXXX) || { echo "Failed to create a temporary file"; exit 1; } + curl -fsSL "$runLinkUrl" -o "$jobArtifactZip" + curlStatus=$? + if [ $curlStatus -ne 0 ] || [ ! -s "$jobArtifactZip" ]; then + echo "Failed to download the build from $runLinkUrl" + exit 1 + fi + jobExtractDir=$(mktemp -d -t firefly-build-extract.XXXXXX) || { echo "Failed to create a temporary directory"; exit 1; } + if ! unzip -oq "$jobArtifactZip" -d "$jobExtractDir" || [ ! -f "$jobExtractDir/standalone.zip" ]; then + echo "Failed to expand the build artifact from $runLinkUrl" + exit 1 + fi + installUrlArgs=(-url "$jobExtractDir/standalone.zip") +fi + +# now execute the real install script +"$downloadedInstallScript" -dontConfirm "${installUrlArgs[@]}" "$@" diff --git a/bin/install.sh b/bin/install.sh index 1fe1440e9e..fe3af971fa 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -36,6 +36,48 @@ isTrue() { if [[ "$v" == "true" || "$v" == "t" ]]; then return 0; else return 1; fi } +# -------------------------- +# checkRequiredCommands: verify the basic tools needed to install are present +# -------------------------- + +checkRequiredCommands() { + missing="" + for cmd in curl unzip realpath; do + if ! command -v "$cmd" > /dev/null 2>&1; then + missing="$missing $cmd" + fi + done + if [ -n "$missing" ]; then + echo "Cannot install: the following required command(s) are missing:$missing" + echo "Please install them and re-run this script." + exit 1 + fi +} + +# -------------------------- +# checkOsCompatibility: warn if the OS does not meet the documented requirements +# (see docs/using-firefly-standalone.md) +# -------------------------- + +checkOsCompatibility() { + name=$(uname) + if [[ "$name" == "Darwin" ]]; then + osVersion=$(sw_vers -productVersion 2> /dev/null) + majorVersion=${osVersion%%.*} + if [[ "$majorVersion" =~ ^[0-9]+$ ]] && [ "$majorVersion" -lt 15 ]; then + echo "Warning: Firefly requires macOS 15 or greater, detected macOS ${osVersion:-unknown}" + fi + elif [[ "$name" == "Linux" ]]; then + command -v ldconfig > /dev/null 2>&1 && hasLdconfig="TRUE" + if isTrue $hasLdconfig && ! ldconfig -p 2> /dev/null | grep -q libssl.so.3; then + echo "Warning: libssl.so.3 was not found. Firefly requires libssl.so.3 (Debian 12+, RHEL 9+, Ubuntu 22.04+, Fedora)." + fi + fi +} + +checkRequiredCommands +checkOsCompatibility + # -------------------------- # get the parameters # -------------------------- @@ -91,7 +133,11 @@ fi # -------------------------- if isTrue $confirm && isTrue $initialInstall && [ "$enteredPath" == "" ]; then - read -p "Enter installation directory [${enteredPath:-$defaultInstallRelativePath}]: " enteredPath + read -p "Enter installation directory [${defaultInstallRelativePath}]: " enteredPath + if [ -n "$enteredPath" ]; then + mkdir -p "$enteredPath" + INSTALL_DIR=$(realpath "$enteredPath") + fi fi @@ -130,19 +176,24 @@ rm -f "$applicationDir"/complete JQ=$(which jq) if [[ "$JQ" == '' ]]; then name=$(uname) - if [[ "$name" == "Darwin" ]]; then - echo jq is is missing from mac os, install failed - exit 1 - fi arch=$(uname -m) - if [[ "$arch" == "x86_64" ]]; then + if [[ "$name" == "Darwin" ]]; then + if [[ "$arch" == "arm64" ]]; then + jqUrl="https://github.com/jqlang/jq/releases/latest/download/jq-macos-arm64" + else + jqUrl="https://github.com/jqlang/jq/releases/latest/download/jq-macos-amd64" + fi + elif [[ "$arch" == "x86_64" ]]; then jqUrl="https://github.com/jqlang/jq/releases/latest/download/jq-linux-amd64" else jqUrl="https://github.com/jqlang/jq/releases/latest/download/jq-linux-arm64" fi echo "installing local jq..." - curl -sL "$jqUrl" -o "$binDir/jq" + if ! curl -fsSL "$jqUrl" -o "$binDir/jq" || [[ ! -s "$binDir/jq" ]]; then + echo "Failed to download jq from $jqUrl, install failed" + exit 1 + fi chmod +x "$binDir/jq" JQ="$binDir/jq" fi @@ -155,22 +206,21 @@ fi targetPackageFile="${applicationDir}/standalone.zip" -packageUrl=$(curl -s "https://api.github.com/repos/Caltech-IPAC/firefly/releases/latest" | \ -$JQ -r '.assets[] | [.name, .browser_download_url] | @tsv' | \ -while IFS=$'\t' read -r asset_name download_url; do - if [ "$asset_name" == $PACKAGE_ASSET_NAME ]; then - echo "$download_url" - fi -done) if [ -z "$altUrl" ]; then - url=$packageUrl + releaseJson=$(curl -s "https://api.github.com/repos/Caltech-IPAC/firefly/releases/latest") + apiError=$(echo "$releaseJson" | $JQ -r '.message // empty' 2> /dev/null) + if [ -n "$apiError" ]; then + echo "Error contacting the GitHub API: $apiError" + exit 1 + fi + url=$(echo "$releaseJson" | $JQ -r --arg name "$PACKAGE_ASSET_NAME" '.assets[]? | select(.name == $name) | .browser_download_url') else url=$altUrl fi if [ -z "$url" ]; then echo "No package defined to download, could not find it as a github asset https://github.com/Caltech-IPAC/firefly/releases" - exit 0 + exit 1 fi @@ -180,20 +230,57 @@ fi echo "install from: $url" if [[ "$url" == http* ]]; then - curl -sL "$url" > "${targetPackageFile}" + httpStatus=$(curl -sL -w "%{http_code}" "$url" -o "${targetPackageFile}") + if [[ "$httpStatus" != "200" ]]; then + echo "Failed to download $url (HTTP status $httpStatus)" + exit 1 + fi else + if [ ! -f "$url" ]; then + echo "Package file not found: $url" + exit 1 + fi cp "$url" "${targetPackageFile}" fi +if [[ ! -s "${targetPackageFile}" ]]; then + echo "Downloaded package is empty: ${targetPackageFile}" + exit 1 +fi + echo "expanding firefly $targetPackageFile..." (cd "$applicationDir" && unzip -o "${targetPackageFile}" &> "${applicationDir}/standalone-expand.log") +if [ $? -ne 0 ]; then + echo "Failed to expand $targetPackageFile, see ${applicationDir}/standalone-expand.log" + exit 1 +fi +if [ ! -f "$applicationDir/firefly.war" ]; then + echo "firefly.war not found after expanding $targetPackageFile, see ${applicationDir}/standalone-expand.log" + exit 1 +fi mkdir -p "$applicationDir/firefly-war" echo "expanding firefly.war..." (cd "$applicationDir/firefly-war" && unzip -o "${applicationDir}/firefly.war" &> "${applicationDir}/war-expand.log") +if [ $? -ne 0 ]; then + echo "Failed to expand firefly.war, see ${applicationDir}/war-expand.log" + exit 1 +fi # -------------------------- # make the script executable, put some in correct place # -------------------------- +requiredFiles=("standalone_cleanup.sh" "$startScript" "startFireflyServer.sh" "javaInstaller.sh" "updater.sh") +missingFiles="" +for f in "${requiredFiles[@]}"; do + if [ ! -f "$applicationDir/$f" ]; then + missingFiles="$missingFiles $f" + fi +done +if [ -n "$missingFiles" ]; then + echo "Expected file(s) missing after expanding the package:$missingFiles" + exit 1 +fi + scriptPath=$(realpath "$0") cp "$scriptPath" "$applicationDir/install.sh" chmod 775 "$applicationDir/standalone_cleanup.sh" \ @@ -207,6 +294,15 @@ chmod 775 "$applicationDir/standalone_cleanup.sh" \ cp "$applicationDir/$startScript" "$binDir" chmod +x "$binDir/$startScript" +# -------------------------- +# link ff into ~/.local/bin, creating it if needed, so ff is available +# without editing PATH on systems where ~/.local/bin is already on it +# -------------------------- + +localBinDir="${HOME}/.local/bin" +mkdir -p "$localBinDir" +ln -sf "$binDir/$startScript" "$localBinDir/$startScript" + # -------------------------- # setup default port # -------------------------- @@ -226,6 +322,10 @@ fi if isTrue $installJre; then echo "installing java..." JAVA=$("$applicationDir"/javaInstaller.sh) + if [ $? -ne 0 ] || [ -z "$JAVA" ]; then + echo "Failed to install Java, see error(s) above" + exit 1 + fi fi # -------------------------- @@ -238,7 +338,7 @@ if isTrue $initialInstall; then echo echo ">>>>>>>>>>>>>>>>>>>>>> ${binDir#$PWD/}/ff start" echo - echo "You might want to add the bin dir to your PATH: $binDir" + echo "You might want to add the bin dir to your PATH: $binDir or ~/.local/bin" fi diff --git a/docs/using-firefly-standalone.md b/docs/using-firefly-standalone.md index 4fa8f211de..dc9843fb55 100644 --- a/docs/using-firefly-standalone.md +++ b/docs/using-firefly-standalone.md @@ -5,7 +5,12 @@ Firefly can be installed directly on your macOS or Linux desktop machine. This is a full-featured installation that performs very well when working with local files. - +### Requirements + +- `curl` and `unzip` must be available on your system; the installer checks for these up front and stops with a clear message if either is missing. +- Java is not required beforehand — the installer downloads a compatible Java runtime automatically unless you configure your own (see [Advanced Configuration](#advanced-configuration)). +- See [Confirming firefly will run on your OS](#confirming-firefly-will-run-on-your-os) for OS-specific requirements. + ## Installing Firefly ### Quick install @@ -170,3 +175,13 @@ Check with the following command #### Windows Standalone Firefly is not supported on Windows +--- + +## Troubleshooting + +- **The installer stops with a missing command error**: install the missing tool (`curl` and/or `unzip`) with your system's package manager and re-run the install. +- **"Error contacting the GitHub API" or "No package defined to download"**: this usually means a network issue or that GitHub's API rate limit was hit. Wait a few minutes and try again, or pass a direct URL/path to a `standalone.zip` with `./install.sh -url `. +- **The installer reports a failed download or a failure expanding a package**: re-run the install; if it persists, check your network connection and firewall, or download the release manually from the [Firefly releases page](https://github.com/Caltech-IPAC/firefly/releases) and reinstall with `./install.sh -url `. +- **`ff start` reports the port is in use**: another application is using the configured port. Change the port in `~/.firefly/config.json` or start with `firefly/bin/ff start --port `. +- **Java fails to install automatically**: install Java 21+ yourself and set its path in the `java` field of `~/.firefly/config.json`, replacing `"auto"`. + diff --git a/src/firefly/js/ui/FileUpload.jsx b/src/firefly/js/ui/FileUpload.jsx index 665b44f47f..608935515d 100644 --- a/src/firefly/js/ui/FileUpload.jsx +++ b/src/firefly/js/ui/FileUpload.jsx @@ -51,7 +51,11 @@ const ChooseUploadFile= ({onChange, value, fileName, canDragDrop}) => (