From eaab75985d2a99ea6a4263fbaed4a42121cfca1e Mon Sep 17 00:00:00 2001 From: Titouan Rigoudy Date: Tue, 11 Aug 2026 03:59:30 +0200 Subject: [PATCH] Make google-cloud-cli apt package installation and docker repo installation conditional on rodete When running local environment setup (./local/install_deps.bash) on internal Google Linux workstations (rodete), apt-get fails when attempting to install several optional App Engine and Pub/Sub component packages that are not provided in internal apt repositories. Furthermore, attempting to add the docker-ce repository triggers interactive glogin authentication even when docker is already installed on the system. Specifically: - google-cloud-cli-app-engine-python, google-cloud-cli-app-engine-python-extras, google-cloud-cli-app-engine-go, and google-cloud-cli-pubsub-emulator apt packages are optional/unavailable on rodete (see internal references b/414408644 and b/484368884). - glogin and glinux-add-repo docker-ce-rodete should be skipped if docker is already installed on the machine. - Fix pipenv CLI invocation in local/install_python_deps_linux.bash. This commit updates local/install_deps_linux.bash to check $distro_codename and whether docker is installed. On rodete, only google-cloud-cli and google-cloud-cli-datastore-emulator are installed via apt-get, while non-rodete distributions retain the full component package list. Additionally adds local/tests/install_deps_test_linux.bash to test and verify fresh checkout setup on Linux. --- local/install_deps_linux.bash | 27 +++++--- local/install_python_deps_linux.bash | 3 + local/tests/install_deps_test_linux.bash | 80 ++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 9 deletions(-) create mode 100755 local/tests/install_deps_test_linux.bash diff --git a/local/install_deps_linux.bash b/local/install_deps_linux.bash index 74cbcb71d7b..e690ad93fe7 100755 --- a/local/install_deps_linux.bash +++ b/local/install_deps_linux.bash @@ -90,8 +90,10 @@ sudo apt-get install -y \ sudo apt-get install -y apt-transport-https software-properties-common if [ "$distro_codename" == "rodete" ]; then - glogin - sudo glinux-add-repo docker-ce-"$distro_codename" + if ! which docker > /dev/null 2>&1; then + glogin + sudo glinux-add-repo docker-ce-"$distro_codename" + fi else curl -fsSL https://download.docker.com/linux/${distro_id,,}/gpg | \ sudo apt-key add - @@ -147,13 +149,20 @@ if gcloud components install --quiet beta; then else # Either Cloud SDK component manager is disabled (default on GCE), or google-cloud-cli package is # installed via apt-get. - sudo apt-get install -y \ - google-cloud-cli-app-engine-go \ - google-cloud-cli-app-engine-python \ - google-cloud-cli-app-engine-python-extras \ - google-cloud-cli \ - google-cloud-cli-datastore-emulator \ - google-cloud-cli-pubsub-emulator + # Note: app-engine-python, app-engine-python-extras, and pubsub-emulator apt packages are optional on rodete (b/414408644, b/484368884). + if [ "$distro_codename" == "rodete" ]; then + sudo apt-get install -y \ + google-cloud-cli \ + google-cloud-cli-datastore-emulator + else + sudo apt-get install -y \ + google-cloud-cli \ + google-cloud-cli-app-engine-go \ + google-cloud-cli-app-engine-python \ + google-cloud-cli-app-engine-python-extras \ + google-cloud-cli-datastore-emulator \ + google-cloud-cli-pubsub-emulator + fi fi dir=$(dirname "$0") diff --git a/local/install_python_deps_linux.bash b/local/install_python_deps_linux.bash index 6a4db1c9dfe..e9d736618e6 100755 --- a/local/install_python_deps_linux.bash +++ b/local/install_python_deps_linux.bash @@ -54,6 +54,9 @@ fi # Install other dependencies (e.g. bower). nodeenv -p --prebuilt +# Re-activate virtual environment after nodeenv modifies activate scripts and PATH +# so that global node npm packages (like bower) install into the virtual environment bin directory. +source "$(${PYTHON} -m pipenv --venv)/bin/activate" # Unsafe perm flag allows bower and polymer-bundler install for root users as well. npm install --unsafe-perm -g bower polymer-bundler bower --allow-root install diff --git a/local/tests/install_deps_test_linux.bash b/local/tests/install_deps_test_linux.bash new file mode 100755 index 00000000000..7f41e90256c --- /dev/null +++ b/local/tests/install_deps_test_linux.bash @@ -0,0 +1,80 @@ +#!/bin/bash -ex +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "$(uname)" != "Linux" ]; then + echo "install_deps_test_linux.bash is supported on Linux only." + exit 0 +fi + +# Store repository root directory before cloning. +REPO_DIR=$(git rev-parse --show-toplevel) + +TEMP_DIR=$(mktemp -d -t clusterfuzz-fresh-install-XXXXXX) +cleanup() { + rm -rf "$TEMP_DIR" +} +trap cleanup EXIT + +# Clone a clean copy of the repo to test installation in an isolated fresh checkout. +git clone "$REPO_DIR" "$TEMP_DIR/clusterfuzz" +cd "$TEMP_DIR/clusterfuzz" + +# Verify initial clean state (no vendored directories). +if [ -d src/third_party ]; then + echo "ERROR: src/third_party should not exist in fresh checkout." + exit 1 +fi +if [ -d src/appengine/third_party ]; then + echo "ERROR: src/appengine/third_party should not exist in fresh checkout." + exit 1 +fi + +# Run installation script for fresh setup. +./local/install_deps.bash + +# Verify virtual environment and python installation. +VENV_DIR=$(python3 -m pipenv --venv) +if [ ! -f "$VENV_DIR/bin/python" ]; then + echo "ERROR: Pipenv virtual environment python was not created." + exit 1 +fi + +# Verify vendored core third_party packages. +if [ ! -d src/third_party/google/cloud/monitoring_v3 ]; then + echo "ERROR: src/third_party/google/cloud/monitoring_v3 missing." + exit 1 +fi + +# Verify vendored appengine third_party packages. +if [ ! -d src/appengine/third_party/flask ]; then + echo "ERROR: src/appengine/third_party/flask missing." + exit 1 +fi + +# Verify bower frontend components. +if [ ! -d src/appengine/private/bower_components ]; then + echo "ERROR: src/appengine/private/bower_components missing." + exit 1 +fi + +# Verify linting passes in fresh environment. +pipenv run python butler.py lint + +# Verify running core and appengine unit tests passes in fresh environment. +pipenv run python butler.py py_unittest -t core -p deploy_test.py +pipenv run python butler.py py_unittest -t appengine -p home_test.py + +echo "SUCCESS: Fresh checkout setup using local/install_deps.bash verified!"