Update Monitoring, Multiprocessing, and Storage backends

.github/workflows/ibm-ce-tests.yml

@@ -12,7 +12,7 @@ jobs:
 
   determine_runnable_test_jobs:
     runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
+    timeout-minutes: 5
 
     outputs:
       localhost: ${{ steps.script.outputs.localhost }}
@@ -23,20 +23,22 @@ jobs:
         id: script
         run: |
           echo "localhost=true" >> $GITHUB_OUTPUT
-          echo "code_engine=false" >> $GITHUB_OUTPUT
+          # The Code Engine job needs the config secret to run at all
+          echo "code_engine=$HAVE_LITHOPS_CONFIG" >> $GITHUB_OUTPUT
 
 
   localhost_tests:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     needs: determine_runnable_test_jobs
     if: needs.determine_runnable_test_jobs.outputs.localhost == 'true'
 
     steps:
       - name: Clone Lithops repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Install Python 3.10
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.10'
 
@@ -47,20 +49,21 @@ jobs:
       - name: Run Lithops tests
         run: |
           cd lithops/tests
-          pytest -v --backend localhost --storage localhost
+          pytest -v --timeout=120 --timeout-method=thread --backend localhost --storage localhost
 
 
   ibm_ce_cos_tests:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     needs: determine_runnable_test_jobs
     if: needs.determine_runnable_test_jobs.outputs.code_engine == 'true'
 
     steps:
       - name: Clone Lithops repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Install Python 3.10
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.10'
 
@@ -75,7 +78,7 @@ jobs:
 
       - name: Build new runtime
         run: |
-          docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_TOKEN }}
+          echo '${{ secrets.DOCKER_TOKEN }}' | docker login -u '${{ secrets.DOCKER_USER }}' --password-stdin
           cd runtime/code_engine
           lithops runtime build -f Dockerfile.githubci ${{ secrets.DOCKER_USER }}/lithops-ce-gihub-ci:${{ github.run_id }} -b code_engine
           sed -i '/runtime: lithops-ce/c\    runtime: '${{ secrets.DOCKER_USER }}'/lithops-ce-gihub-ci:'${{ github.run_id }} $LITHOPS_CONFIG_FILE

.github/workflows/python-linting.yml

@@ -7,21 +7,27 @@ on:
     paths:
       - 'setup.py'
       - 'lithops/**'
+      - '.github/workflows/python-linting.yml'
 
   workflow_dispatch: 
   # this allows to run the workflow manually through the github dashboard
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
 
   flake8:
     runs-on: ubuntu-latest
+    timeout-minutes: 10
 
     steps:
       - name: Clone Lithops repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Install Python 3.10
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.10'
 
@@ -30,10 +36,6 @@ jobs:
           python3 -m pip install --upgrade pip
           pip3 install -U flake8
 
-      - name: Install Lithops
-        run: |
-          pip3 install -U .
-
       - name: Lint with flake8
         run: |
-          flake8 lithops --count --max-line-length=180 --statistics --ignore W605,W503
+          flake8 lithops --count --statistics

.github/workflows/tests-all-os.yml

@@ -6,60 +6,31 @@ on:
 jobs:
   localhost_tests:
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 6
+    timeout-minutes: 20
     env:
       OBJC_DISABLE_INITIALIZE_FORK_SAFETY: YES
 
     strategy:
-      fail-fast: False
+      fail-fast: false
       matrix:
-        include:
-          # Linux
-          - os: ubuntu-latest
-            python-version: "3.10"
-          - os: ubuntu-latest
-            python-version: "3.11"
-          - os: ubuntu-latest
-            python-version: "3.12"
-          - os: ubuntu-latest
-            python-version: "3.13"
-          - os: ubuntu-22.04
-            python-version: "3.10"
-          - os: ubuntu-22.04
-            python-version: "3.11"
-
-          # macOS
-          - os: macos-latest
-            python-version: "3.10"
-          - os: macos-latest
-            python-version: "3.11"
-          - os: macos-latest
-            python-version: "3.12"
-          - os: macos-latest
-            python-version: "3.13"
-          - os: macos-15
-            python-version: "3.10"
-          - os: macos-15
-            python-version: "3.11"
-
-          # Windows
-          - os: windows-latest
-            python-version: "3.10"
-          - os: windows-latest
-            python-version: "3.11"
-          - os: windows-latest
-            python-version: "3.12"
-          - os: windows-latest
-            python-version: "3.13"
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
 
     steps:
       - name: Clone Lithops repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Install Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: setup.py
+
+      - name: Install Redis
+        uses: shogo82148/actions-setup-redis@v1
+        with:
+          redis-version: '7.x'
 
       - name: Install Lithops
         run: |
@@ -78,8 +49,7 @@ jobs:
       - name: Run Lithops tests
         run: |
           cd lithops/tests
-          # pytest -v --durations=0 --backend localhost --storage localhost
-          pytest -v --durations=0 -o log_cli=true --log-cli-level=DEBUG --backend localhost --storage localhost
+          pytest -v --durations=0 --timeout=120 --timeout-method=thread --backend localhost --storage localhost
 
       - name: Display last 500 lines of the Lithops log file
         if: cancelled() || failure()

.github/workflows/tests.yml

@@ -7,15 +7,32 @@ on:
     paths:
       - 'setup.py'
       - 'lithops/**'
+      - '.github/workflows/tests.yml'
 
   workflow_dispatch: 
   # this allows to run the workflow manually through the github dashboard
 
+# A new push to a PR supersedes the run that is still going for it
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
 
   localhost_tests:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 5
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+
+    services:
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 10
 
     strategy:
       fail-fast: false
@@ -24,12 +41,14 @@ jobs:
 
     steps:
       - name: Clone Lithops repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Install Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: setup.py
 
       - name: Install Lithops
         run: |
@@ -38,4 +57,4 @@ jobs:
       - name: Run Lithops tests
         run: |
           cd lithops/tests
-          pytest -v --backend localhost --storage localhost
+          pytest -v --timeout=120 --timeout-method=thread --backend localhost --storage localhost