diff --git a/local/install_deps_linux.bash b/local/install_deps_linux.bash index 74cbcb71d7b..e690ad93fe7 100755 --- a/local/install_deps_linux.bash +++ b/local/install_deps_linux.bash @@ -90,8 +90,10 @@ sudo apt-get install -y \ sudo apt-get install -y apt-transport-https software-properties-common if [ "$distro_codename" == "rodete" ]; then - glogin - sudo glinux-add-repo docker-ce-"$distro_codename" + if ! which docker > /dev/null 2>&1; then + glogin + sudo glinux-add-repo docker-ce-"$distro_codename" + fi else curl -fsSL https://download.docker.com/linux/${distro_id,,}/gpg | \ sudo apt-key add - @@ -147,13 +149,20 @@ if gcloud components install --quiet beta; then else # Either Cloud SDK component manager is disabled (default on GCE), or google-cloud-cli package is # installed via apt-get. - sudo apt-get install -y \ - google-cloud-cli-app-engine-go \ - google-cloud-cli-app-engine-python \ - google-cloud-cli-app-engine-python-extras \ - google-cloud-cli \ - google-cloud-cli-datastore-emulator \ - google-cloud-cli-pubsub-emulator + # Note: app-engine-python, app-engine-python-extras, and pubsub-emulator apt packages are optional on rodete (b/414408644, b/484368884). + if [ "$distro_codename" == "rodete" ]; then + sudo apt-get install -y \ + google-cloud-cli \ + google-cloud-cli-datastore-emulator + else + sudo apt-get install -y \ + google-cloud-cli \ + google-cloud-cli-app-engine-go \ + google-cloud-cli-app-engine-python \ + google-cloud-cli-app-engine-python-extras \ + google-cloud-cli-datastore-emulator \ + google-cloud-cli-pubsub-emulator + fi fi dir=$(dirname "$0") diff --git a/local/install_python_deps_linux.bash b/local/install_python_deps_linux.bash index 6a4db1c9dfe..e9d736618e6 100755 --- a/local/install_python_deps_linux.bash +++ b/local/install_python_deps_linux.bash @@ -54,6 +54,9 @@ fi # Install other dependencies (e.g. bower). nodeenv -p --prebuilt +# Re-activate virtual environment after nodeenv modifies activate scripts and PATH +# so that global node npm packages (like bower) install into the virtual environment bin directory. +source "$(${PYTHON} -m pipenv --venv)/bin/activate" # Unsafe perm flag allows bower and polymer-bundler install for root users as well. npm install --unsafe-perm -g bower polymer-bundler bower --allow-root install diff --git a/local/tests/install_deps_test_linux.bash b/local/tests/install_deps_test_linux.bash new file mode 100755 index 00000000000..7f41e90256c --- /dev/null +++ b/local/tests/install_deps_test_linux.bash @@ -0,0 +1,80 @@ +#!/bin/bash -ex +# +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "$(uname)" != "Linux" ]; then + echo "install_deps_test_linux.bash is supported on Linux only." + exit 0 +fi + +# Store repository root directory before cloning. +REPO_DIR=$(git rev-parse --show-toplevel) + +TEMP_DIR=$(mktemp -d -t clusterfuzz-fresh-install-XXXXXX) +cleanup() { + rm -rf "$TEMP_DIR" +} +trap cleanup EXIT + +# Clone a clean copy of the repo to test installation in an isolated fresh checkout. +git clone "$REPO_DIR" "$TEMP_DIR/clusterfuzz" +cd "$TEMP_DIR/clusterfuzz" + +# Verify initial clean state (no vendored directories). +if [ -d src/third_party ]; then + echo "ERROR: src/third_party should not exist in fresh checkout." + exit 1 +fi +if [ -d src/appengine/third_party ]; then + echo "ERROR: src/appengine/third_party should not exist in fresh checkout." + exit 1 +fi + +# Run installation script for fresh setup. +./local/install_deps.bash + +# Verify virtual environment and python installation. +VENV_DIR=$(python3 -m pipenv --venv) +if [ ! -f "$VENV_DIR/bin/python" ]; then + echo "ERROR: Pipenv virtual environment python was not created." + exit 1 +fi + +# Verify vendored core third_party packages. +if [ ! -d src/third_party/google/cloud/monitoring_v3 ]; then + echo "ERROR: src/third_party/google/cloud/monitoring_v3 missing." + exit 1 +fi + +# Verify vendored appengine third_party packages. +if [ ! -d src/appengine/third_party/flask ]; then + echo "ERROR: src/appengine/third_party/flask missing." + exit 1 +fi + +# Verify bower frontend components. +if [ ! -d src/appengine/private/bower_components ]; then + echo "ERROR: src/appengine/private/bower_components missing." + exit 1 +fi + +# Verify linting passes in fresh environment. +pipenv run python butler.py lint + +# Verify running core and appengine unit tests passes in fresh environment. +pipenv run python butler.py py_unittest -t core -p deploy_test.py +pipenv run python butler.py py_unittest -t appengine -p home_test.py + +echo "SUCCESS: Fresh checkout setup using local/install_deps.bash verified!" diff --git a/src/clusterfuzz/_internal/cron/chrome_tests_syncer.py b/src/clusterfuzz/_internal/cron/chrome_tests_syncer.py index c01833a535a..5c0bc0595aa 100644 --- a/src/clusterfuzz/_internal/cron/chrome_tests_syncer.py +++ b/src/clusterfuzz/_internal/cron/chrome_tests_syncer.py @@ -55,10 +55,12 @@ def unpack_crash_testcases(crash_testcases_directory): # uploaded repros. Check if the testcase is fixed. If not, skip. # Only use testcases that have bugs associated with them. # Sort latest first. + # We use `> ''` instead of `!= ''` because the Datastore emulator does not + # support gRPC NOT_EQUAL (operator 9) filters. testcases = data_types.Testcase.query( ndb_utils.is_false( data_types.Testcase.open), data_types.Testcase.status == 'Processed', - data_types.Testcase.bug_information != + data_types.Testcase.bug_information > '').order(-data_types.Testcase.timestamp) for testcase in testcases: count += 1 diff --git a/src/clusterfuzz/_internal/cron/fuzzer_and_job_weights.py b/src/clusterfuzz/_internal/cron/fuzzer_and_job_weights.py index dd4cbea9f4e..c9ad248af37 100644 --- a/src/clusterfuzz/_internal/cron/fuzzer_and_job_weights.py +++ b/src/clusterfuzz/_internal/cron/fuzzer_and_job_weights.py @@ -314,9 +314,12 @@ def update_target_weights_for_engine(client, engine, specifications): # All fuzzers with non-default weights must be tracked with a special # specification. This ensures that they will be restored to normal weight # once conditions causing adjustments are no longer met. + # Range inequality filters (< / >) are used instead of `!= 1.0` because the + # Datastore emulator does not support gRPC NOT_EQUAL (operator 9) filters. target_jobs = data_types.FuzzTargetJob.query( - data_types.FuzzTarget.engine == engine).filter( - data_types.FuzzTargetJob.weight != 1.0) + data_types.FuzzTargetJob.engine == engine).filter( + ndb.OR(data_types.FuzzTargetJob.weight < 1.0, + data_types.FuzzTargetJob.weight > 1.0)) for target_job in target_jobs: matches[(target_job.fuzz_target_name, diff --git a/src/clusterfuzz/_internal/cron/oss_fuzz_apply_ccs.py b/src/clusterfuzz/_internal/cron/oss_fuzz_apply_ccs.py index fa856cd0f7e..c20cafb3869 100644 --- a/src/clusterfuzz/_internal/cron/oss_fuzz_apply_ccs.py +++ b/src/clusterfuzz/_internal/cron/oss_fuzz_apply_ccs.py @@ -24,10 +24,12 @@ def get_open_testcases_with_bugs(): """Return iterator to open testcases with bugs.""" + # We use `> ''` instead of `!= ''` because the Datastore emulator does not + # support gRPC NOT_EQUAL (operator 9) filters. return data_types.Testcase.query( ndb_utils.is_true(data_types.Testcase.open), data_types.Testcase.status == 'Processed', - data_types.Testcase.bug_information != '').order( # pylint: disable=g-explicit-bool-comparison + data_types.Testcase.bug_information > '').order( data_types.Testcase.bug_information, data_types.Testcase.key) diff --git a/src/clusterfuzz/_internal/cron/retry_stuck_tasks.py b/src/clusterfuzz/_internal/cron/retry_stuck_tasks.py index 240e86d8ef7..752979efb66 100644 --- a/src/clusterfuzz/_internal/cron/retry_stuck_tasks.py +++ b/src/clusterfuzz/_internal/cron/retry_stuck_tasks.py @@ -144,6 +144,7 @@ def _get_stuck_testcase_candidates_query(stuck_deadline: datetime.datetime, in 'Processed' or 'Duplicate' status and those not marked as 'NA' for the 'fixed' property, although this last filter has significant performance implications. + in 'Processed' or 'Duplicate' status. Args: stuck_deadline: The datetime threshold. Testcases updated more recently @@ -152,8 +153,11 @@ def _get_stuck_testcase_candidates_query(stuck_deadline: datetime.datetime, Returns: An ndb.Query object for the candidate testcases. """ + # Range inequality filters (< / >) are used instead of `!= 'NA'` because the + # Datastore emulator does not support gRPC NOT_EQUAL (operator 9) filters. return data_types.Testcase.query( - data_types.Testcase.fixed != 'NA', + ndb.OR(data_types.Testcase.fixed < 'NA', + data_types.Testcase.fixed > 'NA'), ndb_utils.is_false(data_types.Testcase.one_time_crasher_flag), ndb_utils.is_true(data_types.Testcase.open), ndb.OR(data_types.Testcase.status == 'Processed',