From 2e0b044293b509d745f3cb58a6ccf158086b837f Mon Sep 17 00:00:00 2001 From: Bug2 Repro Date: Thu, 28 May 2026 14:50:29 -0700 Subject: [PATCH 1/3] Bug #2 repro: diagnose Python venv label vs runtime version mismatch Sets platformVersion: 3.12 (as the getting-started.md docs say to do) and runs diagnose.sh which prints: - what /output/pythonenv* directories exist - whether /output/pythonenv3.12/bin/python (the #840 docs path) actually works - what sys.version / sys.executable / sys.prefix the runtime actually has - whether C-extension .so files match the runtime ABI Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build.yaml | 6 +++-- diagnose.sh | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100755 diagnose.sh diff --git a/build.yaml b/build.yaml index 4aea474..c17572e 100644 --- a/build.yaml +++ b/build.yaml @@ -1,5 +1,7 @@ version: 1 platform: python -platformVersion: "3.14" +platformVersion: "3.12" + run: - port: 8080 \ No newline at end of file + port: 8080 + startCommand: "bash /output/diagnose.sh" diff --git a/diagnose.sh b/diagnose.sh new file mode 100755 index 0000000..697635e --- /dev/null +++ b/diagnose.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Bug #2 diagnostic — proves the build/runtime Python version split. +# Runs at app-start time, inside the runtime container. + +set +e # don't bail on first error — we want to see every failure mode + +echo "==============================================" +echo "=== BUG-2 DIAGNOSTIC :: STARTED $(date -u) ===" +echo "==============================================" + +echo "" +echo "--- [1] embr.yaml requested platformVersion: 3.12 ---" + +echo "" +echo "--- [2] /output/ directory listing ---" +ls -la /output/ 2>&1 | head -30 + +echo "" +echo "--- [3] /output/pythonenv*/ subdirectories ---" +ls -ld /output/pythonenv* 2>&1 + +echo "" +echo "--- [4] /output/pythonenv3.12/bin/ contents ---" +ls -la /output/pythonenv3.12/bin/ 2>&1 | head -20 + +echo "" +echo "--- [5] Symlink target of /output/pythonenv3.12/bin/python ---" +ls -la /output/pythonenv3.12/bin/python 2>&1 +readlink -f /output/pythonenv3.12/bin/python 2>&1 +echo "Does it actually exist?" +test -x /output/pythonenv3.12/bin/python && echo "YES, executable" || echo "NO, broken or missing" + +echo "" +echo "--- [6] Try running /output/pythonenv3.12/bin/python --version (what #840 docs say to use) ---" +/output/pythonenv3.12/bin/python --version 2>&1 +echo "Exit code from above: $?" + +echo "" +echo "--- [7] Runtime python on PATH ---" +which python python3 2>&1 +python --version 2>&1 +python3 --version 2>&1 + +echo "" +echo "--- [8] sys.version / sys.executable / sys.prefix (proves what user actually got) ---" +python3 -c "import sys; print('version:', sys.version); print('executable:', sys.executable); print('prefix:', sys.prefix); print('base_prefix:', sys.base_prefix)" 2>&1 + +echo "" +echo "--- [9] lib/python*/site-packages subtrees under each pythonenv* ---" +find /output -maxdepth 4 -type d -name 'site-packages' 2>&1 + +echo "" +echo "--- [10] sample C-extension wheel ABI check ---" +echo "If a 3.12-built wheel landed in pythonenv3.12/lib/python3.12/site-packages," +echo "is it findable from the 3.14 runtime?" +find /output -name '*.cpython-312*.so' 2>&1 | head -5 +find /output -name '*.cpython-314*.so' 2>&1 | head -5 + +echo "" +echo "==============================================" +echo "=== BUG-2 DIAGNOSTIC :: COMPLETE =============" +echo "==============================================" + +# Start a minimal healthcheck server so the deploy stays up and we can re-read logs. +# Don't use the venv path on purpose — if [6] above showed it's broken, this is our fallback. +exec python3 -c " +from http.server import HTTPServer, BaseHTTPRequestHandler +class H(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200); self.end_headers(); self.wfile.write(b'ok') + def log_message(self, *a): pass +print('[diag] healthcheck server listening on :8080', flush=True) +HTTPServer(('0.0.0.0', 8080), H).serve_forever() +" From 670e6d3ab769e3470e1f26e9e6d0fccefedfc275 Mon Sep 17 00:00:00 2001 From: Bug2 Repro Date: Thu, 28 May 2026 14:53:54 -0700 Subject: [PATCH 2/3] Bug #2 repro v2: drop platformVersion (original FRICTIONS condition); probe failure modes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.yaml b/build.yaml index c17572e..506fb99 100644 --- a/build.yaml +++ b/build.yaml @@ -1,6 +1,7 @@ version: 1 platform: python -platformVersion: "3.12" +# NOTE: platformVersion intentionally omitted — this is the FRICTIONS-observed bug condition. +# Per static analysis, this should: build with whatever Oryx auto-picks, runtime selects default (3.14). run: port: 8080 From 244eb8d0594e1121a642e5b238c7438cea59a8db Mon Sep 17 00:00:00 2001 From: Bug2 Repro Date: Thu, 28 May 2026 14:54:16 -0700 Subject: [PATCH 3/3] Bug #2 repro v2: actually update diagnose.sh with failure-mode probes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- diagnose.sh | 89 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/diagnose.sh b/diagnose.sh index 697635e..4b20cb5 100755 --- a/diagnose.sh +++ b/diagnose.sh @@ -1,74 +1,85 @@ #!/bin/bash -# Bug #2 diagnostic — proves the build/runtime Python version split. -# Runs at app-start time, inside the runtime container. - -set +e # don't bail on first error — we want to see every failure mode - +set +e echo "==============================================" -echo "=== BUG-2 DIAGNOSTIC :: STARTED $(date -u) ===" +echo "=== BUG-2 DIAGNOSTIC v2 :: $(date -u) ========" echo "==============================================" echo "" -echo "--- [1] embr.yaml requested platformVersion: 3.12 ---" +echo "--- [A] Enumerate every python* binary in the runtime image ---" +ls -la /usr/bin/python* /usr/local/bin/python* /opt/*/bin/python* 2>&1 | sort -u +echo "PATH=$PATH" echo "" -echo "--- [2] /output/ directory listing ---" -ls -la /output/ 2>&1 | head -30 +echo "--- [B] pyvenv.cfg of the shipped venv ---" +cat /app/pythonenv3.12/pyvenv.cfg 2>&1 || cat /output/pythonenv3.12/pyvenv.cfg 2>&1 echo "" -echo "--- [3] /output/pythonenv*/ subdirectories ---" -ls -ld /output/pythonenv* 2>&1 +echo "--- [C] file & ldd of the venv's python binary ---" +file /app/pythonenv3.12/bin/python 2>&1 || file /output/pythonenv3.12/bin/python 2>&1 +echo "----" +ldd /app/pythonenv3.12/bin/python 2>&1 | head -10 || ldd /output/pythonenv3.12/bin/python 2>&1 | head -10 echo "" -echo "--- [4] /output/pythonenv3.12/bin/ contents ---" -ls -la /output/pythonenv3.12/bin/ 2>&1 | head -20 +echo "--- [D] Try the path Oryx's runtime banner advertised ---" +echo "Banner said: PYTHONPATH=':/output/pythonenv3.12/lib/python3.14/site-packages'" +ls -d /output/pythonenv3.12/lib/python3.14/site-packages 2>&1 +test -d /output/pythonenv3.12/lib/python3.14/site-packages && echo "EXISTS" || echo "DOES NOT EXIST" echo "" -echo "--- [5] Symlink target of /output/pythonenv3.12/bin/python ---" -ls -la /output/pythonenv3.12/bin/python 2>&1 -readlink -f /output/pythonenv3.12/bin/python 2>&1 -echo "Does it actually exist?" -test -x /output/pythonenv3.12/bin/python && echo "YES, executable" || echo "NO, broken or missing" +echo "--- [E] FAILURE MODE 1: user runs system python3.14 with banner's PYTHONPATH ---" +if command -v python3.14 >/dev/null 2>&1; then + PYTHONPATH=/output/pythonenv3.12/lib/python3.14/site-packages python3.14 -c "import fastapi; print('imported fastapi from', fastapi.__file__)" 2>&1 + echo "Exit: $?" +else + echo "(no python3.14 in this image)" +fi echo "" -echo "--- [6] Try running /output/pythonenv3.12/bin/python --version (what #840 docs say to use) ---" -/output/pythonenv3.12/bin/python --version 2>&1 -echo "Exit code from above: $?" +echo "--- [F] FAILURE MODE 2: docs invariant reversed for platformVersion: 3.14 ---" +ls -d /output/pythonenv3.14 2>&1 +test -d /output/pythonenv3.14 && echo "EXISTS" || echo "DOES NOT EXIST (the dir is named from build-time, so 3.14 venv would only exist when platformVersion=3.14)" echo "" -echo "--- [7] Runtime python on PATH ---" -which python python3 2>&1 -python --version 2>&1 -python3 --version 2>&1 +echo "--- [G] FAILURE MODE 3: 3.12-built C-extension under 3.14 ABI interpreter ---" +echo "Find cp312 pydantic_core .so:" +find /output /app -name '_pydantic_core.cpython-3*.so' 2>/dev/null | head -5 +if command -v python3.14 >/dev/null 2>&1; then + echo "Try loading cp312 wheel under python3.14:" + PYTHONPATH=/app/pythonenv3.12/lib/python3.12/site-packages python3.14 -c "import pydantic_core; print(pydantic_core.__version__)" 2>&1 + echo "Exit: $?" +fi echo "" -echo "--- [8] sys.version / sys.executable / sys.prefix (proves what user actually got) ---" -python3 -c "import sys; print('version:', sys.version); print('executable:', sys.executable); print('prefix:', sys.prefix); print('base_prefix:', sys.base_prefix)" 2>&1 +echo "--- [H] Oryx's runtime startup script (what generated the misleading banner) ---" +ls -la /app/oryx-startup.sh /output/oryx-startup.sh 2>&1 +cat /app/oryx-startup.sh 2>&1 | head -60 || cat /output/oryx-startup.sh 2>&1 | head -60 echo "" -echo "--- [9] lib/python*/site-packages subtrees under each pythonenv* ---" -find /output -maxdepth 4 -type d -name 'site-packages' 2>&1 +echo "--- [I] What runtime image are we in? ---" +cat /etc/os-release 2>&1 | head -5 +echo "---" +test -f /opt/oryx/python_version_info && cat /opt/oryx/python_version_info +test -f /usr/local/oryx/python_version_info && cat /usr/local/oryx/python_version_info echo "" -echo "--- [10] sample C-extension wheel ABI check ---" -echo "If a 3.12-built wheel landed in pythonenv3.12/lib/python3.12/site-packages," -echo "is it findable from the 3.14 runtime?" -find /output -name '*.cpython-312*.so' 2>&1 | head -5 -find /output -name '*.cpython-314*.so' 2>&1 | head -5 +echo "--- [J] Confirm the basics from v1 still hold ---" +echo "which python: $(which python 2>&1)" +echo "python --version: $(python --version 2>&1)" +echo "/output/pythonenv3.12/bin/python --version: $(/output/pythonenv3.12/bin/python --version 2>&1)" +echo "sys.executable / sys.version:" +python -c "import sys; print(sys.executable); print(sys.version)" 2>&1 echo "" echo "==============================================" -echo "=== BUG-2 DIAGNOSTIC :: COMPLETE =============" +echo "=== BUG-2 DIAGNOSTIC v2 :: COMPLETE ==========" echo "==============================================" -# Start a minimal healthcheck server so the deploy stays up and we can re-read logs. -# Don't use the venv path on purpose — if [6] above showed it's broken, this is our fallback. -exec python3 -c " +exec python -c " from http.server import HTTPServer, BaseHTTPRequestHandler class H(BaseHTTPRequestHandler): def do_GET(self): self.send_response(200); self.end_headers(); self.wfile.write(b'ok') def log_message(self, *a): pass -print('[diag] healthcheck server listening on :8080', flush=True) +print('[diag v2] healthcheck server listening on :8080', flush=True) HTTPServer(('0.0.0.0', 8080), H).serve_forever() "