Migrate dependency management from Pipenv to uv

.devcontainer/devcontainer.json

@@ -23,6 +23,6 @@
       }
     }
   },
-  "postCreateCommand": "python --version && java --version && pip install --upgrade pip && pip install pipenv && bash local/install_deps.bash",
+  "postCreateCommand": "python --version && java --version && bash local/install_deps.bash",
   "remoteUser": "vscode"
 }

.gitignore

@@ -26,6 +26,7 @@
 /local/bin/android-sdk/*
 /local/storage
 /paramiko.log
+/.venv
 /src/appengine/requirements.txt
 /src/appengine/app.yaml
 /src/appengine/cron-service.yaml
@@ -43,7 +44,6 @@
 /src/local/remote/generated.remmina
 /src/protos/ENV
 /src/third_party
-/src/requirements.txt
 _test_data
 bazel-*
 /src/dist
@@ -57,5 +57,5 @@ bazel-*
 **/.terraform/*
 
 # Ignore temporary build files.
-docker/base/Pipfile
-docker/base/Pipfile.lock
+docker/base/pyproject.toml
+docker/base/uv.lock

AGENTS.md

@@ -38,15 +38,7 @@ asdf plugin add gcloud https://github.com/jthegedus/asdf-gcloud
 asdf install
 ```
 
-### 4. Install `pipenv`
-
-Once `asdf` has installed the correct python version, you need to install `pipenv`:
-
-```bash
-python -m pip install pipenv
-```
-
-### 5. Install Project Dependencies
+### 4. Install Project Dependencies
 
 Now you are ready to install the project's dependencies. Run the following command from the root of the repository:
 
@@ -55,18 +47,17 @@ Now you are ready to install the project's dependencies. Run the following comma
 ```
 
 ## Using Butler
-Before each time you use the `butler.py` script, verify that you are inside the virtual environment, if not activate it by running `python -m pipenv shell`, or pre append any `python butler.py` call with
-`pipenv run`
+Before using the `butler.py` script, verify that you are inside the virtual environment (`source .venv/bin/activate`), or prepend any `butler.py` call with `uv run`.
 
 For instance:
 ```bash
-pipenv run python butler.py lint
+uv run butler.py lint
 ```
 Is the same as running
 ```bash
 python butler.py lint
 ```
-if you are inside a virtual environment.
+if you are inside an activated virtual environment.
 
 ## Testing
 

cli/casp/src/casp/utils/container.py

@@ -32,7 +32,7 @@
 # The base command prefix for executing ClusterFuzz butler commands.
 # This ensures that commands are run with the correct Python environment
 # and logging settings within the container.
-_COMMAND_PREFIX = 'pipenv run python butler.py --local-logging'
+_COMMAND_PREFIX = 'uv run butler.py --local-logging'
 
 
 def build_butler_command(subcommand: str, **kwargs: str) -> list[str]:

cloudbuild.yaml

@@ -29,27 +29,14 @@ steps:
     mv tmp/.git .git
     rm -rf tmp
 - name: 'gcr.io/clusterfuzz-images/ci:d41d82f-202601161634'
-  args: ['pipenv', 'sync', '--dev', '--python=3.11']
-  env:
-    - PIPENV_VENV_IN_PROJECT=1
-- name: 'gcr.io/clusterfuzz-images/ci:d41d82f-202601161634'
-  args: ['pipenv', 'run', 'setup']
-  env:
-    - PIPENV_VENV_IN_PROJECT=1
-- name: 'gcr.io/clusterfuzz-images/ci:d41d82f-202601161634'
-  args: ['pipenv', 'run', 'python', 'butler.py', 'lint']
-  env:
-    - PIPENV_VENV_IN_PROJECT=1
-- name: 'gcr.io/clusterfuzz-images/ci:d41d82f-202601161634'
-  args: ['pipenv', 'run', 'local/tests/run_all_tests']
+  args: ['local/tests/ci_tests_inner.bash', 'local/tests/run_all_tests']
   env:
     - TEST_BLOBS_BUCKET=clusterfuzz-ci-blobs
     - TEST_BUCKET=clusterfuzz-ci-test
     - TEST_CORPUS_BUCKET=clusterfuzz-ci-corpus
     - TEST_QUARANTINE_BUCKET=clusterfuzz-ci-quarantine
     - TEST_BACKUP_BUCKET=clusterfuzz-ci-backup
     - TEST_COVERAGE_BUCKET=clusterfuzz-ci-coverage
-    - PIPENV_VENV_IN_PROJECT=1
 timeout: 5400s
 options:
   machineType: E2_HIGHCPU_8

configs/test/bot/setup/android.bash

@@ -96,16 +96,21 @@ else
 fi
 unzip -q clusterfuzz-source.zip
 
-echo "Installing ClusterFuzz package dependencies using pipenv."
+echo "Installing ClusterFuzz package dependencies using uv."
 cd clusterfuzz
 if ! python3 -m pip > /dev/null ; then
   curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
   python3 get-pip.py
 fi
-python3 -m pip install --upgrade pipenv
-pipenv --python 3.11
-pipenv sync
-source "$(pipenv --venv)/bin/activate"
+if ! command -v pipx &> /dev/null; then
+  python3 -m pip install --upgrade pipx==1.10.0
+fi
+if ! command -v uv &> /dev/null; then
+  pipx install uv==0.12.3
+fi
+export PATH="$PATH:$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")"
+uv sync
+source .venv/bin/activate
 
 if [ -z "$ANDROID_SERIAL" ]; then
   echo "No \$ANDROID_SERIAL set. Will automatically detect devices and start ClusterFuzz for each."

configs/test/bot/setup/linux.bash

@@ -91,15 +91,20 @@ else
 fi
 unzip -q clusterfuzz-source.zip
 
-echo "Installing ClusterFuzz package dependencies using pipenv."
+echo "Installing ClusterFuzz package dependencies using uv."
 cd clusterfuzz
 if ! python3 -m pip > /dev/null ; then
   curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
   python3 get-pip.py
 fi
-python3 -m pip install --upgrade pipenv
-pipenv --python 3.11
-pipenv sync
-source "$(pipenv --venv)/bin/activate"
+if ! command -v pipx &> /dev/null; then
+  python3 -m pip install --upgrade pipx==1.10.0
+fi
+if ! command -v uv &> /dev/null; then
+  pipx install uv==0.12.3
+fi
+export PATH="$PATH:$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")"
+uv sync
+source .venv/bin/activate
 
 run_bot &

configs/test/bot/setup/mac.bash

@@ -84,16 +84,21 @@ else
 fi
 unzip -q clusterfuzz-source.zip
 
-echo "Installing ClusterFuzz package dependencies using pipenv."
+echo "Installing ClusterFuzz package dependencies using uv."
 cd clusterfuzz
 if ! python3 -m pip > /dev/null ; then
   curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
   python3 get-pip.py
 fi
-python3 -m pip install --upgrade pipenv
-pipenv --python 3.11
-pipenv sync
-source "$(pipenv --venv)/bin/activate"
+if ! command -v pipx &> /dev/null; then
+  python3 -m pip install --upgrade pipx==1.10.0
+fi
+if ! command -v uv &> /dev/null; then
+  pipx install uv==0.12.3
+fi
+export PATH="$PATH:$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$HOME/.local/bin")"
+uv sync
+source .venv/bin/activate
 
 echo "Running ClusterFuzz."
 GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS" ROOT_DIR="$ROOT_DIR" PYTHONPATH="$PYTHONPATH" GSUTIL_PATH="$GSUTIL_PATH" python $ROOT_DIR/src/python/bot/startup/run.py &

configs/test/gce/windows-init.ps1

@@ -172,7 +172,9 @@ cmd /c c:\python27\python -m pip install -U wheel
 cmd /c c:\python27\python -m pip install crcmod==1.7 cryptography==3.3.2 pyOpenSSL==17.4.0 pywinauto==0.6.4 psutil==5.4.7 future==0.17.1
 
 cmd /c c:\python311\python -m pip install -U pip
-cmd /c c:\python311\python -m pip install pipenv
+cmd /c c:\python311\python -m pip install pipx==1.10.0
+cmd /c c:\python311\python -m pipx ensurepath
+cmd /c c:\python311\python -m pipx install uv==0.12.3
 
 # Install NodeJS.
 $fileName = "$tmp\nodejs.zip"
@@ -301,13 +303,11 @@ Set-Content $packageSetupFilePath "Skipped package install"
 # Schedule chkdsk on every reboot.
 echo y | chkdsk C: /F /I /C
 
-# Install Pipfile dependencies
-$env:Path += ";c:\python311;c:\python311\scripts"
+# Install dependencies using uv
+$pipxBinDir = cmd /c c:\python311\python -m pipx environment --value PIPX_BIN_DIR
+$env:Path += ";c:\python311;c:\python311\scripts;$pipxBinDir"
 cd c:\clusterfuzz
-cmd /c c:\python311\scripts\pipenv install --deploy --system
-
-# Can't be managed by pipenv due to https://github.com/pypa/pipenv/issues/3193.
-cmd /c c:\python311\python -m pip install pywinauto==0.6.8
+cmd /c uv sync
 
 # Run the scripts.
 Write-Host "Run scripts"

deploy/cloudbuild.yaml

@@ -66,9 +66,11 @@ steps:
 
     # Install required deps for performing butler deploy
     bash ./local/install_deps.bash
-
+
+    export PATH="$$PATH:$$(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "$$HOME/.local/bin")"
+
     # Get into the venv
-    source "$$(python3.11 -m pipenv --venv)/bin/activate"
+    source .venv/bin/activate
 
     # Check if the _PROJECT substitution was provided.
     if [ -z "${_PROJECT}" ]; then

docker/base/.gitignore

@@ -1,2 +1,2 @@
-Pipfile
-Pipfile.lock
+pyproject.toml
+uv.lock

docker/base/Dockerfile

@@ -124,7 +124,11 @@ RUN curl -sS https://www.python.org/ftp/python/3.11.4/Python-3.11.4.tgz | tar -C
     cd /tmp/Python-3.11.4 && \
     ./configure --enable-optimizations --enable-loadable-sqlite-extensions && make altinstall && \
     rm -rf /tmp/Python-3.11.4 /tmp/Python-3.11.4.tar.xz
-RUN pip3.11 --no-cache-dir install pipenv==2022.8.5
+# Symlink uv binary installed by pipx to /usr/local/bin so it is on system PATH,
+# and symlink python3.11 to /usr/bin/python3.11 for system tools expecting it in /usr/bin.
+RUN pip3.11 --no-cache-dir install pipx==1.10.0
+RUN pipx install uv==0.12.3
+RUN ln -s $(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "/root/.local/bin")/uv /usr/local/bin/uv
 RUN ln -s /usr/local/bin/python3.11 /usr/bin/python3.11
 
 # Install Node.js
@@ -168,10 +172,10 @@ RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV PYTHONIOENCODING UTF-8
 
-COPY setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh Pipfile Pipfile.lock start.sh /data/
+COPY setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh pyproject.toml uv.lock start.sh /data/
 RUN cd /data && \
-    # Make pip3.11 the default so that pipenv install --system works.
+    # Make pip3.11 the default so that pip install works.
     mv /usr/local/bin/pip3.11 /usr/local/bin/pip && \
-    python3.11 -m pipenv install --deploy --system
+    uv pip install --system -e /data
 
 CMD ["bash", "-ex", "/data/start.sh"]

docker/base/ubuntu-20-04.Dockerfile

@@ -124,7 +124,11 @@ RUN curl -sS https://www.python.org/ftp/python/3.11.4/Python-3.11.4.tgz | tar -C
     cd /tmp/Python-3.11.4 && \
     ./configure --enable-optimizations --enable-loadable-sqlite-extensions && make altinstall && \
     rm -rf /tmp/Python-3.11.4 /tmp/Python-3.11.4.tar.xz
-RUN pip3.11 --no-cache-dir install pipenv==2022.8.5
+# Symlink uv binary installed by pipx to /usr/local/bin so it is on system PATH,
+# and symlink python3.11 to /usr/bin/python3.11 for system tools expecting it in /usr/bin.
+RUN pip3.11 --no-cache-dir install pipx==1.10.0
+RUN pipx install uv==0.12.3
+RUN ln -s $(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "/root/.local/bin")/uv /usr/local/bin/uv
 RUN ln -s /usr/local/bin/python3.11 /usr/bin/python3.11
 
 # Install Node.js
@@ -168,10 +172,10 @@ RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV PYTHONIOENCODING UTF-8
 
-COPY setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh Pipfile Pipfile.lock start.sh /data/
+COPY setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh pyproject.toml uv.lock start.sh /data/
 RUN cd /data && \
-    # Make pip3.11 the default so that pipenv install --system works.
+    # Make pip3.11 the default so that pip install works.
     mv /usr/local/bin/pip3.11 /usr/local/bin/pip && \
-    python3.11 -m pipenv install --deploy --system
+    uv pip install --system -e /data
 
 CMD ["bash", "-ex", "/data/start.sh"]

docker/base/ubuntu-24-04.Dockerfile

@@ -100,7 +100,9 @@ RUN curl -sS https://www.python.org/ftp/python/3.11.4/Python-3.11.4.tgz | tar -C
     cd /tmp/Python-3.11.4 && \
     ./configure --enable-optimizations --enable-loadable-sqlite-extensions && make altinstall && \
     rm -rf /tmp/Python-3.11.4 /tmp/Python-3.11.4.tar.xz
-RUN pip3.11 --no-cache-dir install pipenv==2022.8.5
+RUN pip3.11 --no-cache-dir install pipx==1.10.0
+RUN pipx install uv==0.12.3
+RUN ln -s $(pipx environment --value PIPX_BIN_DIR 2>/dev/null || echo "/root/.local/bin")/uv /usr/local/bin/uv
 RUN ln -s /usr/local/bin/python3.11 /usr/bin/python3.11 && \
     ln -s /usr/bin/python3.11 /usr/bin/python3
 
@@ -149,10 +151,10 @@ ENV PYTHONIOENCODING UTF-8
 ENV BASE_OS_VERSION=ubuntu-24-04
 
 
-COPY Pipfile Pipfile.lock setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh start.sh /data/
+COPY pyproject.toml uv.lock setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh start.sh /data/
 RUN cd /data && \
-    # Make pip3.11 the default so that pipenv install --system works.
+    # Make pip3.11 the default so that pip install works.
     mv /usr/local/bin/pip3.11 /usr/local/bin/pip && \
-    python3.11 -m pipenv install --deploy --system
+    uv pip install --system -e /data
 
 CMD ["bash", "-ex", "/data/start.sh"]

docker/build.sh

@@ -32,12 +32,12 @@ IMAGES=(
 VERSION_TAG="latest"
 GIT_HASH_ARG=""
 PUSH="true"
-NEEDS_ROOT_PIPFILE=false
+NEEDS_ROOT_PYPROJECT=false
 
-# Set up a trap to clean up Pipfiles on exit.
+# Set up a trap to clean up pyproject files on exit.
 function cleanup() {
-  if [[ "$NEEDS_ROOT_PIPFILE" == "true" ]]; then
-    rm -f base/Pipfile base/Pipfile.lock
+  if [[ "$NEEDS_ROOT_PYPROJECT" == "true" ]]; then
+    rm -f base/pyproject.toml base/uv.lock
   fi
 }
 trap cleanup EXIT
@@ -102,19 +102,19 @@ for image_name in "${IMAGES[@]}"; do
   image_with_version_tag="$image_name:$VERSION_TAG"
   image_with_stamp="$image_name:$stamp"
 
-  # Copy Pipfile to base for the ubuntu-24.04 build, as it's
+  # Copy pyproject.toml to base for the ubuntu-24.04 build, as it's
   # needed but not in the build context.
   if [[ "$image_dir" == "base" && "$dockerfile" == *"ubuntu-24-04"* ]]; then
-    NEEDS_ROOT_PIPFILE=true
-    cp ../Pipfile ../Pipfile.lock base/
+    NEEDS_ROOT_PYPROJECT=true
+    cp ../pyproject.toml ../uv.lock base/
   fi
 
   docker build -t "$image_with_version_tag" -f "$dockerfile" "$image_dir"
 
   # Clean up the copied files.
-  if [[ "$NEEDS_ROOT_PIPFILE" == "true" ]]; then
-    rm base/Pipfile base/Pipfile.lock
-    NEEDS_ROOT_PIPFILE=false
+  if [[ "$NEEDS_ROOT_PYPROJECT" == "true" ]]; then
+    rm base/pyproject.toml base/uv.lock
+    NEEDS_ROOT_PYPROJECT=false
   fi
 
   docker tag "$image_with_version_tag" "$image_with_stamp"

docker/build_on_container_builder.sh

@@ -24,5 +24,5 @@ if [[ -n "$git_diff" ]]; then
   exit 1
 fi
 
-cp ../Pipfile* base/
+cp ../pyproject.toml ../uv.lock base/
 gcloud builds submit . --project=clusterfuzz-images --substitutions=_GIT_HASH=$(git rev-parse HEAD | head -c7)