diff --git a/.changes/unreleased/+portable-python-interpreter-evidence-v2.yaml b/.changes/unreleased/+portable-python-interpreter-evidence-v2.yaml new file mode 100644 index 00000000..58c9a43c --- /dev/null +++ b/.changes/unreleased/+portable-python-interpreter-evidence-v2.yaml @@ -0,0 +1,2 @@ +kind: Changed +body: Record complete isolated Python interpreter compatibility facts before resolving packages. diff --git a/docs/.review/APT_PROVIDER.md b/docs/.review/APT_PROVIDER.md index 660818db..7955f885 100644 --- a/docs/.review/APT_PROVIDER.md +++ b/docs/.review/APT_PROVIDER.md @@ -1,26 +1,27 @@ --- artifact: swe-design-review-attestation -schema_version: 2 -scope_key: bd8b79396ed3ab3d8e062279e409430bcf54daf5acf561e1b5d63f4302f9edc4 -scope: {"kind": "path", "primary_target": "docs/APT_PROVIDER.md", "repository": "/home/omry/dev/reploy", "selector": "docs/APT_PROVIDER.md"} -review_content_identity_sha256: 3cb75520f49dee8255a7bc807e55670c1c59c75981b5753cb3c94ad29749b593 -target_content_identity_sha256: 5c8bcb33496743ccef81f81684cf38482d1a9b225e096f47b328b95ea049a68a -baseline_content_identity_sha256: 9f98a355bf1d27e1c417bb5080a2f86eda476110f89ccf264d5ba02f7ee8d13b -target_documents: [{"path": "docs/APT_PROVIDER.md", "repository": "/home/omry/dev/reploy", "sha256": "be2008134ae557cf275893325fcd7caae7588ced609c5dc05bf142ad0a6aa60c"}] -baseline_documents: [{"path": "docs/BLUEPRINT_ENVIRONMENT_MODEL.md", "repository": "/home/omry/dev/reploy", "sha256": "8969ffde4d2f20d2e02fb530d0e9688ed63d9dd8962a9a700473d4a6a54e7ef4"}] -document_repository: "/home/omry/dev/reploy" +schema_version: 4 +scope_key: 65b7fe1d45c88c7805c4b9fcd8ce6d6d69dc0e4b962ecdbf253f7e8a872b4662 +scope: {"kind": "path", "primary_target": "docs/APT_PROVIDER.md", "repository": ".", "selector": "docs/APT_PROVIDER.md"} +review_content_identity_sha256: 5c23192cdcbc74302da21994a357b4fe52adee05272cafdf1f4316d69cf29637 +target_content_identity_sha256: 01b5c3c3907d236c5e62773a80426585be85e44b1d86518411b1efe555a1ca15 +baseline_content_identity_sha256: 58468667dc4b7ccc1b53ece6069a0c768862bc3447a74f838ba9c1edfb6a12c9 +target_documents: [{"path": "docs/APT_PROVIDER.md", "repository": ".", "sha256": "b6d439338beb432953f9dae5a00dda7db78340c2309778b44ce0d920b8e873d3"}] +baseline_documents: [{"path": "docs/BLUEPRINT_ENVIRONMENT_MODEL.md", "repository": ".", "sha256": "9c9f8af518952cdea721ebdaea548930ee5456833d618eb54735b2c0431badd3"}] +design_dependency_documents: [{"path": "docs/BLUEPRINT_ENVIRONMENT_MODEL.md", "repository": ".", "sha256": "9c9f8af518952cdea721ebdaea548930ee5456833d618eb54735b2c0431badd3"}] +document_repository: "." document_path: "docs/APT_PROVIDER.md" -document_revision_provenance: "a13e17402b578d5f6c6922ac82773ad2f116237f" -document_sha256: be2008134ae557cf275893325fcd7caae7588ced609c5dc05bf142ad0a6aa60c +document_revision_provenance: "9affd2736d3767b4ea9847fb76585e218c805470" +document_sha256: b6d439338beb432953f9dae5a00dda7db78340c2309778b44ce0d920b8e873d3 verdict: clean -attested_at: 2026-08-22T19:51:20Z +attested_at: 2026-09-15T07:02:11Z --- - + # SWE design-review attestation Review freshness is determined by the target and baseline document bytes -listed in the version-2 header. Revisions are provenance only. +listed in the version-4 header. Revisions are provenance only. ## Durable review state diff --git a/docs/.review/APT_PROVIDER_DETAIL_DESIGN.md b/docs/.review/APT_PROVIDER_DETAIL_DESIGN.md index cbcf3b74..ef376617 100644 --- a/docs/.review/APT_PROVIDER_DETAIL_DESIGN.md +++ b/docs/.review/APT_PROVIDER_DETAIL_DESIGN.md @@ -1,26 +1,27 @@ --- artifact: swe-design-review-attestation -schema_version: 2 -scope_key: dd04dd2c2041a5035ffd15d245763f28fd28d9c4d7c4e83ca3a0cb45f092a863 -scope: {"kind": "path", "primary_target": "docs/APT_PROVIDER_DETAIL_DESIGN.md", "repository": "/home/omry/dev/reploy", "selector": "docs/APT_PROVIDER_DETAIL_DESIGN.md"} -review_content_identity_sha256: 8ba6a9a8660c148eaa26c1de35dd900e98f0d6e274c9cfb4d0b1036fb17f56ff -target_content_identity_sha256: e93421059c4ba62f015aa08c248901201e6f091a26fdee58614dbf6e684456bb -baseline_content_identity_sha256: 3f6f4ef49e1b4270a8aefc5120f8d86e3837e74390ea648dc09505d2047f53f4 -target_documents: [{"path": "docs/APT_PROVIDER_DETAIL_DESIGN.md", "repository": "/home/omry/dev/reploy", "sha256": "14e7809c811f921168594cf40aae0c2e9d480016352d59d74fcaa8030c12bdda"}] -baseline_documents: [{"path": "docs/APT_PROVIDER.md", "repository": "/home/omry/dev/reploy", "sha256": "be2008134ae557cf275893325fcd7caae7588ced609c5dc05bf142ad0a6aa60c"}, {"path": "docs/BLUEPRINT_ENVIRONMENT_MODEL.md", "repository": "/home/omry/dev/reploy", "sha256": "8969ffde4d2f20d2e02fb530d0e9688ed63d9dd8962a9a700473d4a6a54e7ef4"}] -document_repository: "/home/omry/dev/reploy" +schema_version: 4 +scope_key: 084819e0d337bf7258eb3a959a310688f60b87260bbfd23042ea7eae5370e31b +scope: {"kind": "path", "primary_target": "docs/APT_PROVIDER_DETAIL_DESIGN.md", "repository": ".", "selector": "docs/APT_PROVIDER_DETAIL_DESIGN.md"} +review_content_identity_sha256: a024823502d76bc7edcd697f32274800c1d03f1473f09e70cd41e2c60a071d85 +target_content_identity_sha256: 740b68117c56cbfa733e0751383b9a0cbb19f8865892c8d989c3c6b36964831f +baseline_content_identity_sha256: d6436371829ec92e56af5bfa51c9de8ca604088f352b2b6322d444cf11f86164 +target_documents: [{"path": "docs/APT_PROVIDER_DETAIL_DESIGN.md", "repository": ".", "sha256": "dbfc92872e7e3a456797eda67e37732f11e3299a9ee7b0543850db18f223b676"}] +baseline_documents: [{"path": "docs/APT_PROVIDER.md", "repository": ".", "sha256": "b6d439338beb432953f9dae5a00dda7db78340c2309778b44ce0d920b8e873d3"}, {"path": "docs/BLUEPRINT_ENVIRONMENT_MODEL.md", "repository": ".", "sha256": "9c9f8af518952cdea721ebdaea548930ee5456833d618eb54735b2c0431badd3"}] +design_dependency_documents: [{"path": "docs/APT_PROVIDER.md", "repository": ".", "sha256": "b6d439338beb432953f9dae5a00dda7db78340c2309778b44ce0d920b8e873d3"}, {"path": "docs/BLUEPRINT_ENVIRONMENT_MODEL.md", "repository": ".", "sha256": "9c9f8af518952cdea721ebdaea548930ee5456833d618eb54735b2c0431badd3"}] +document_repository: "." document_path: "docs/APT_PROVIDER_DETAIL_DESIGN.md" -document_revision_provenance: "a13e17402b578d5f6c6922ac82773ad2f116237f" -document_sha256: 14e7809c811f921168594cf40aae0c2e9d480016352d59d74fcaa8030c12bdda +document_revision_provenance: "9affd2736d3767b4ea9847fb76585e218c805470" +document_sha256: dbfc92872e7e3a456797eda67e37732f11e3299a9ee7b0543850db18f223b676 verdict: clean -attested_at: 2026-08-22T19:52:51Z +attested_at: 2026-09-15T07:19:52Z --- - + # SWE design-review attestation Review freshness is determined by the target and baseline document bytes -listed in the version-2 header. Revisions are provenance only. +listed in the version-4 header. Revisions are provenance only. ## Durable review state diff --git a/docs/APT_PROVIDER.md b/docs/APT_PROVIDER.md index 88d7b8fa..61a977d3 100644 --- a/docs/APT_PROVIDER.md +++ b/docs/APT_PROVIDER.md @@ -1,6 +1,6 @@ --- status: Active -updated: 2026-08-22 +updated: 2026-09-15 summary: Subdesign for closed .deb package layers, provider outputs, and Python runtime dependencies. refines: docs/BLUEPRINT_ENVIRONMENT_MODEL.md --- @@ -1502,16 +1502,25 @@ does not enumerate link groups, choose an alternative, or accept an unregistered alternatives link. Both commands run inside the already-required consuming or final-validation container. -The consuming provider then applies semantic validation as the first operation -in its disposable bundle resolver. For Python, it can execute: +The consuming provider applies semantic validation as the first operation in +its disposable bundle resolver. For Python, it invokes the selected absolute +interpreter with a fixed isolated command prefix: ```text -/usr/bin/python3 -I -S -c - import sys; print(".".join(map(str, sys.version_info[:3]))) +/usr/bin/python3 -I -c ``` -It then checks the requested Python version constraint and separately verifies -that the interpreter can create a virtual environment. +The provider-controlled environment and working directory remain in force. +System-site initialization stays enabled so the probe and ordinary +`python -I -m pip` resolver import the same selected interpreter's installed +pip tag generator, while isolated mode excludes current-directory, user-site, +and environment-controlled import shadows. The bounded tested-tag set and +selected architecture are validated data, never executable source. The strict +result records the complete Python version, implementation, ABI, libc identity +and release, the exact tested tags, and their compatible subset. The probe does +no network work, artifact acquisition, or wheel parsing. The provider then +checks the requested Python version constraint and separately verifies that the +interpreter can create a virtual environment. Package and logical versions are distinct: diff --git a/docs/APT_PROVIDER_DETAIL_DESIGN.md b/docs/APT_PROVIDER_DETAIL_DESIGN.md index d8ce5e9f..56564a40 100644 --- a/docs/APT_PROVIDER_DETAIL_DESIGN.md +++ b/docs/APT_PROVIDER_DETAIL_DESIGN.md @@ -1,6 +1,6 @@ --- status: Active -updated: 2026-08-22 +updated: 2026-09-15 summary: Implemented local-Docker design for the provider graph, APT/dpkg bundles, generated image layers, and cross-provider executable consumption. implements: docs/APT_PROVIDER.md --- @@ -1603,8 +1603,8 @@ Environment names match `[A-Za-z_][A-Za-z0-9_]*`, are unique and sorted, and the profile always sets `InheritNone`. `Umask` is four lowercase octal digits. Secrets are not transaction fields. -The initial Python transaction uses recipe `python-materialize-v1` and child -environment `python-v1` with `InheritNone=true`, `Umask=0022`, and no variables. +The Python transaction uses recipe `python-materialize-v2` and child environment +`python-v2` with `InheritNone=true`, `Umask=0022`, and no variables. It runs as numeric root from `/` with `NetworkPolicy=none`. Its read-only script mount is keyed by the provider-owned script digest; its read-only wheel mount is keyed by the complete resolved-bundle identity. The selected interpreter is a @@ -1960,13 +1960,21 @@ commands. For the built-in `python3` mapping, the APT provider publishes only the singleton candidate path and mapping provenance. The consuming Python node validates it as the first step inside its existing bundle-resolver container, -before network or source work. Its typed adapter uses fixed `-I -S -c` arguments -and provider-owned code to require a Python implementation and parse its actual -version, ABI, and platform. A missing path, non-Python executable, or -unparseable version fails with the mapping identity and an example using -`exports.python.executable`; the resolver performs no path search or fallback -discovery. The materializer proves `venv` support by creating the real component -venv; it does not create a disposable venv first. +before network or source work. Its typed adapter invokes the selected absolute +interpreter with fixed `-I -c` arguments under the provider-owned clean +environment and working directory. System-site initialization remains enabled +so the probe can import the selected interpreter's installed +`pip._vendor.packaging.tags`; isolated mode still excludes current-directory, +user-site, and environment-controlled import shadows. A bounded, sorted, unique +tested-tag set and selected architecture are passed as validated data, never +interpolated into the provider-owned probe. The strict V2 result records the +complete Python version, implementation, ABI, libc identity and release, the +exact tested tags, and their compatible subset. The probe performs no path +search, fallback discovery, network access, acquisition, or wheel parsing. A +missing path, non-Python executable, missing installed pip, or malformed facts +fails with the mapping identity and an example using +`exports.python.executable`. The materializer proves `venv` support by creating +the real component venv; it does not create a disposable venv first. Python then resolves/builds wheels in a disposable resolver container based on that exact upstream prefix. An exact complete Python bundle hit skips the diff --git a/internal/dockerdeploy/full_validation_python_profile.go b/internal/dockerdeploy/full_validation_python_profile.go index 73df0b84..d265e129 100644 --- a/internal/dockerdeploy/full_validation_python_profile.go +++ b/internal/dockerdeploy/full_validation_python_profile.go @@ -63,26 +63,30 @@ func validatePythonProfileObservation( return providers.ExecutableEvidence{}, fmt.Errorf("validate Python image profile interpreter before execution: %w", err) } - version, err := session.runPythonInterpreterInspection(ctx, locked.InvocationPath) + lockedFacts, err := pythonprovider.DecodeInterpreterFactsV2(locked.Facts) + if err != nil { + return providers.ExecutableEvidence{}, fmt.Errorf("validate Python image profile interpreter facts: %w", err) + } + facts, err := session.runPythonInterpreterInspection(ctx, locked.InvocationPath, lockedFacts.TestedTags) if err != nil { return providers.ExecutableEvidence{}, fmt.Errorf( "Python interpreter at %s is not usable; configure an explicit Python interpreter executable path: %w", locked.InvocationPath, err, ) } - matches, err := pythonprovider.InterpreterVersionSatisfies(requirement.VersionConstraint, version) + matches, err := pythonprovider.InterpreterVersionSatisfies(requirement.VersionConstraint, facts.Version) if err != nil { return providers.ExecutableEvidence{}, err } if !matches { return providers.ExecutableEvidence{}, fmt.Errorf( "Python interpreter at %s has version %s, which does not satisfy %q; configure an explicit Python interpreter executable path", - locked.InvocationPath, version, requirement.VersionConstraint, + locked.InvocationPath, facts.Version, requirement.VersionConstraint, ) } fresh, err := ExecutableEvidenceFromProbe(interpreterObservation, ProbeExecutableBinding{ Requirement: &requirement, Output: locked.Output, - Facts: pythonprovider.CanonicalInterpreterFactsV1(version), + Facts: pythonprovider.CanonicalInterpreterFactsV2(facts), }) if err != nil { return providers.ExecutableEvidence{}, fmt.Errorf("validate Python image profile interpreter: %w", err) @@ -90,19 +94,27 @@ func validatePythonProfileObservation( return fresh, nil } -func (session *ImageValidationSession) runPythonInterpreterInspection(ctx context.Context, interpreterPath string) (string, error) { +func (session *ImageValidationSession) runPythonInterpreterInspection( + ctx context.Context, + interpreterPath string, + testedTags []string, +) (pythonprovider.InterpreterInspectionFactsV2, error) { if session == nil || session.closed { - return "", fmt.Errorf("image validation session is not open") + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("image validation session is not open") } if ctx == nil { - return "", fmt.Errorf("image validation Python inspection context is required") + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("image validation Python inspection context is required") } if err := ctx.Err(); err != nil { - return "", fmt.Errorf("run image validation Python inspection: %w", err) + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("run image validation Python inspection: %w", err) + } + targetArchitecture, err := pythonInspectionArchitectureV2(session.descriptor.Platform) + if err != nil { + return pythonprovider.InterpreterInspectionFactsV2{}, err } - inspection, err := pythonprovider.InterpreterInspectionArgv(interpreterPath) + inspection, err := pythonprovider.InterpreterInspectionArgv(interpreterPath, testedTags, targetArchitecture) if err != nil { - return "", err + return pythonprovider.InterpreterInspectionFactsV2{}, err } args := []string{ "exec", "--user", "0:0", "--workdir", "/", session.containerName, @@ -116,7 +128,7 @@ func (session *ImageValidationSession) runPythonInterpreterInspection(ctx contex if err := session.runDockerCommand(CommandSpec{Name: "docker", Args: args}, RunOptions{ Context: ctx, Stdout: &stdout, Stderr: &stderr, }); err != nil { - return "", imageValidationCommandError("Python interpreter inspection", session.descriptor.Platform.Canonical, stderr.String(), err) + return pythonprovider.InterpreterInspectionFactsV2{}, imageValidationCommandError("Python interpreter inspection", session.descriptor.Platform.Canonical, stderr.String(), err) } - return pythonprovider.ParseInterpreterInspectionOutput(stdout.Bytes()) + return pythonprovider.ParseInterpreterInspectionOutput(stdout.Bytes(), testedTags) } diff --git a/internal/dockerdeploy/full_validation_python_profile_test.go b/internal/dockerdeploy/full_validation_python_profile_test.go index 3375cdfb..a7f4a4aa 100644 --- a/internal/dockerdeploy/full_validation_python_profile_test.go +++ b/internal/dockerdeploy/full_validation_python_profile_test.go @@ -16,9 +16,9 @@ func TestValidatePythonProfileObservationRunsFixedInspectionInHeldSession(t *tes input := completion.Validation.Final profile := input.Profiles[0] locked := profile.SelectedExecutables[0] - version, ok := locked.Facts.Value["version"].(string) - if !ok { - t.Fatalf("locked interpreter facts = %#v", locked.Facts) + lockedFacts, err := pythonprovider.DecodeInterpreterFactsV2(locked.Facts) + if err != nil { + t.Fatal(err) } launcher := directExecutableObservation("shared_launcher", pythonLauncherPath) interpreter := directExecutableObservation("shared_interpreter", locked.InvocationPath) @@ -29,7 +29,7 @@ func TestValidatePythonProfileObservationRunsFixedInspectionInHeldSession(t *tes commands := []CommandSpec{} runImageValidationFollowupCommand = func(spec CommandSpec, options RunOptions) error { commands = append(commands, spec) - _, _ = options.Stdout.Write([]byte(version + "\n")) + _, _ = options.Stdout.Write(pythonInspectionOutputFromFactsV2ForTest(t, locked.Facts)) return nil } session := &ImageValidationSession{descriptor: input.Image.Descriptor, containerName: "held-validation"} @@ -37,10 +37,14 @@ func TestValidatePythonProfileObservationRunsFixedInspectionInHeldSession(t *tes if err != nil { t.Fatal(err) } - if fresh.Facts.Value["version"] != version || fresh.Terminal.Size != "2" { + if fresh.Facts.Value["version"] != lockedFacts.Version || fresh.Terminal.Size != "2" { t.Fatalf("fresh interpreter evidence = %#v", fresh) } - inspection, err := pythonprovider.InterpreterInspectionArgv(locked.InvocationPath) + targetArchitecture, err := pythonInspectionArchitectureV2(input.Image.Descriptor.Platform) + if err != nil { + t.Fatal(err) + } + inspection, err := pythonprovider.InterpreterInspectionArgv(locked.InvocationPath, lockedFacts.TestedTags, targetArchitecture) if err != nil { t.Fatal(err) } @@ -61,7 +65,6 @@ func TestValidatePythonProfileObservationRejectsRequestDriftAndIncompatibleVersi input := completion.Validation.Final profile := input.Profiles[0] locked := profile.SelectedExecutables[0] - version := locked.Facts.Value["version"].(string) launcher := directExecutableObservation("shared_launcher", pythonLauncherPath) interpreter := directExecutableObservation("shared_interpreter", locked.InvocationPath) interpreter.Terminal.Size = "2" @@ -69,7 +72,7 @@ func TestValidatePythonProfileObservationRejectsRequestDriftAndIncompatibleVersi previous := runImageValidationFollowupCommand t.Cleanup(func() { runImageValidationFollowupCommand = previous }) runImageValidationFollowupCommand = func(_ CommandSpec, options RunOptions) error { - _, _ = options.Stdout.Write([]byte(version + "\n")) + _, _ = options.Stdout.Write(pythonInspectionOutputFromFactsV2ForTest(t, locked.Facts)) return nil } session := &ImageValidationSession{descriptor: input.Image.Descriptor, containerName: "held-validation"} @@ -82,7 +85,7 @@ func TestValidatePythonProfileObservationRejectsRequestDriftAndIncompatibleVersi } runImageValidationFollowupCommand = func(_ CommandSpec, options RunOptions) error { - _, _ = options.Stdout.Write([]byte("0.0.0\n")) + _, _ = options.Stdout.Write(pythonInspectionOutputV2ForTest("0.0.0", nil, nil)) return nil } if _, err := validatePythonProfileObservation(context.Background(), session, profile, launcher, interpreter); err == nil || !strings.Contains(err.Error(), "does not satisfy") { diff --git a/internal/dockerdeploy/full_validation_runner_test.go b/internal/dockerdeploy/full_validation_runner_test.go index e4ee7018..8d160187 100644 --- a/internal/dockerdeploy/full_validation_runner_test.go +++ b/internal/dockerdeploy/full_validation_runner_test.go @@ -148,7 +148,7 @@ func TestProviderFullImageValidationRunnerValidatesPythonProfileAndOutputFromOne observations = append(observations, directExecutableObservation(inspection.ID, inspection.InvocationPath)) } response := mustCanonicalProbeResponse(t, probe.ResponseV1{Schema: probe.ResponseSchemaV1, Observations: observations}) - version := input.Profiles[0].SelectedExecutables[0].Facts.Value["version"].(string) + inspectionOutput := pythonInspectionOutputFromFactsV2ForTest(t, input.Profiles[0].SelectedExecutables[0].Facts) commands := []CommandSpec{} runImageValidationFollowupCommand = func(spec CommandSpec, options RunOptions) error { commands = append(commands, spec) @@ -159,7 +159,7 @@ func TestProviderFullImageValidationRunnerValidatesPythonProfileAndOutputFromOne _, _ = options.Stdout.Write(response) return nil } - _, _ = options.Stdout.Write([]byte(version + "\n")) + _, _ = options.Stdout.Write(inspectionOutput) return nil } profiles, outputs, err := (ProviderFullImageValidationRunner{}).Run(context.Background(), input) diff --git a/internal/dockerdeploy/prepared_python_graph_execution.go b/internal/dockerdeploy/prepared_python_graph_execution.go index 1a6da055..f57d1413 100644 --- a/internal/dockerdeploy/prepared_python_graph_execution.go +++ b/internal/dockerdeploy/prepared_python_graph_execution.go @@ -13,6 +13,7 @@ import ( "github.com/omry/reploy/internal/buildprogress" "github.com/omry/reploy/internal/deploy" "github.com/omry/reploy/internal/providers" + pythonprovider "github.com/omry/reploy/internal/providers/python" "github.com/omry/reploy/internal/providers/registry" "github.com/omry/reploy/internal/providerstore" ) @@ -27,6 +28,7 @@ type PreparedPythonGraphExecutionInput struct { Sources []providers.ResolvedSourceInput SourceWheels []providerstore.ArtifactDescriptor LocalOverrides []PythonLocalOverrideV1 + PortablePython *pythonprovider.PortableToolPythonProjectionV1 SourceBuilder *SourceBuilderCoordinatorV1 CurrentLock *deploy.BuildLockV1 FinalImageConfig providers.ImageConfigPolicy @@ -62,8 +64,16 @@ func ExecutePreparedPythonGraph( return providers.GraphExecutionResult{}, err } dropSourceBuilderPythonCachedResolutionsV1(input.Plan, input.CurrentLock, reuse.CachedResolutions) + bindingsByComponent, err := portablePythonProjectionComponentsV1(input.Plan, input.PortablePython) + if err != nil { + return providers.GraphExecutionResult{}, err + } for id, config := range reuse.NodeConfigs { config.LocalOverrides = append([]PythonLocalOverrideV1{}, input.LocalOverrides...) + node, found := graphBackendNode(input.Plan, id) + if found && len(node.Components) == 1 { + config.PortableToolBindings = bindingsByComponent[node.Components[0]] + } config.SourceBuilder = input.SourceBuilder reuse.NodeConfigs[id] = config } @@ -97,6 +107,41 @@ func ExecutePreparedPythonGraph( }) } +func portablePythonProjectionComponentsV1( + plan providers.ProviderPlanV1, + projection *pythonprovider.PortableToolPythonProjectionV1, +) (map[string]*pythonprovider.PortableToolPythonComponentV1, error) { + result := map[string]*pythonprovider.PortableToolPythonComponentV1{} + if projection == nil { + return result, nil + } + if _, err := pythonprovider.CanonicalPortableToolPythonProjectionBytesV1(*projection); err != nil { + return nil, err + } + pythonComponents := map[string]struct{}{} + for _, node := range plan.Nodes { + if node.Provider == blueprint.ComponentTypePython && len(node.Components) == 1 { + pythonComponents[node.Components[0]] = struct{}{} + } + } + for _, component := range projection.Components { + if _, found := pythonComponents[component.Component]; !found { + return nil, fmt.Errorf("portable Python projection component %q has no planned Python node", component.Component) + } + clone := component + clone.TestedTags = append([]string{}, component.TestedTags...) + clone.Bindings = append([]pythonprovider.PortableToolPythonBindingV1{}, component.Bindings...) + for index := range clone.Bindings { + clone.Bindings[index].Requirements = append([]string{}, component.Bindings[index].Requirements...) + clone.Bindings[index].SupportedPython = append([]string{}, component.Bindings[index].SupportedPython...) + clone.Bindings[index].SupportedTags = append([]string{}, component.Bindings[index].SupportedTags...) + clone.Bindings[index].Wheel.Tags = append([]string{}, component.Bindings[index].Wheel.Tags...) + } + result[component.Component] = &clone + } + return result, nil +} + // A source-builder lock proves which portable tools were selected for the // previous build, but the coordinator that prepares those tools is driven by // fresh Python resolution. Preserve reusable wheel candidates while forcing diff --git a/internal/dockerdeploy/prepared_python_graph_execution_test.go b/internal/dockerdeploy/prepared_python_graph_execution_test.go index bc9daa19..b4b74e37 100644 --- a/internal/dockerdeploy/prepared_python_graph_execution_test.go +++ b/internal/dockerdeploy/prepared_python_graph_execution_test.go @@ -10,7 +10,9 @@ import ( "github.com/omry/reploy/internal/blueprint" "github.com/omry/reploy/internal/buildprogress" "github.com/omry/reploy/internal/deploy" + "github.com/omry/reploy/internal/portabletool" "github.com/omry/reploy/internal/providers" + pythonprovider "github.com/omry/reploy/internal/providers/python" "github.com/omry/reploy/internal/providerstore" ) @@ -112,6 +114,12 @@ func TestExecutePreparedPythonGraphDerivesAllReuseFromCurrentLock(t *testing.T) fixture := newPreparedPythonGraphReuseFixture(t) descriptor := fixture.lock.Base localOverrides := []PythonLocalOverrideV1{{Distribution: "demo-server", HostDir: "/tmp/demo-server"}} + node, found := graphBackendNode(fixture.request.Plan, fixture.request.NodeID) + if !found || len(node.Components) != 1 { + t.Fatal("Python fixture node is missing") + } + component := node.Components[0] + portablePython := portablePythonExecutionProjectionForTest(component) previousPrepare := preparePythonGraphExecutionBackend previousExecute := executePreparedPythonProviderGraph t.Cleanup(func() { @@ -142,6 +150,7 @@ func TestExecutePreparedPythonGraphDerivesAllReuseFromCurrentLock(t *testing.T) Store: fixture.store, Plan: fixture.request.Plan, BaseDescriptor: descriptor, BaseCatalog: fixture.request.EarlierCatalog, Sources: fixture.request.SourceCandidates, SourceWheels: fixture.sourceWheels, CurrentLock: &fixture.lock, LocalOverrides: localOverrides, + PortablePython: &portablePython, FinalImageConfig: pythonConsumerTestImageConfig(), }) if err != nil { @@ -156,6 +165,10 @@ func TestExecutePreparedPythonGraphDerivesAllReuseFromCurrentLock(t *testing.T) if !reflect.DeepEqual(configs[fixture.request.NodeID].LocalOverrides, localOverrides) { t.Fatalf("local overrides = %#v", configs[fixture.request.NodeID].LocalOverrides) } + bindings := configs[fixture.request.NodeID].PortableToolBindings + if bindings == nil || bindings.Component != component || !reflect.DeepEqual(bindings.TestedTags, []string{"py3-none-any"}) { + t.Fatalf("portable Python node bindings = %#v", bindings) + } if _, found := execution.CachedResolutions[fixture.request.NodeID]; !found { t.Fatalf("cached resolutions = %#v", execution.CachedResolutions) } @@ -168,6 +181,36 @@ func TestExecutePreparedPythonGraphDerivesAllReuseFromCurrentLock(t *testing.T) } } +func portablePythonExecutionProjectionForTest(component string) pythonprovider.PortableToolPythonProjectionV1 { + closureDigest := rendererDigest("a") + contractDigest := rendererDigest("b") + artifactDigest := rendererDigest("c") + tag := "py3-none-any" + return pythonprovider.PortableToolPythonProjectionV1{ + Schema: pythonprovider.PortableToolPythonProjectionSchemaV1, + Components: []pythonprovider.PortableToolPythonComponentV1{{ + Component: component, TestedTags: []string{tag}, + Bindings: []pythonprovider.PortableToolPythonBindingV1{{ + Scope: "application:application", Component: component, + SelectedClosureDigest: closureDigest, Distribution: "demo-server", + Contract: providers.PortableToolRecordReferenceV1{ + ID: "tool:demo/releases/1.0.0/bindings/python/contract", Digest: contractDigest, + }, + Artifact: providers.PortableToolRecordReferenceV1{ + ID: "tool:demo/releases/1.0.0/bindings/python/artifacts/linux-amd64", Digest: artifactDigest, + }, + Requirements: []string{"demo-server==1.0"}, SupportedPython: []string{"3.13"}, SupportedTags: []string{tag}, + CLI: portabletool.ToolExportV1{Name: "demo", Path: "/opt/demo/bin/demo"}, + Wheel: pythonprovider.PortableToolExactWheelConstraintV1{ + Platform: "linux/amd64", Filename: "demo_server-1.0-py3-none-any.whl", + Distribution: "demo-server", EcosystemVersion: "1.0", Tags: []string{tag}, + Size: "1", SHA256: artifactDigest, RequiresPython: ">=3.13,<3.14", + }, + }}, + }}, + } +} + func TestExecutePreparedPythonGraphRetainsScratchAfterExecutionFailure(t *testing.T) { fixture := newPreparedPythonGraphReuseFixture(t) previousPrepare := preparePythonGraphExecutionBackend diff --git a/internal/dockerdeploy/prepared_python_graph_reuse_test.go b/internal/dockerdeploy/prepared_python_graph_reuse_test.go index 6ed01e16..b5e1cfa7 100644 --- a/internal/dockerdeploy/prepared_python_graph_reuse_test.go +++ b/internal/dockerdeploy/prepared_python_graph_reuse_test.go @@ -401,7 +401,7 @@ func newPreparedPythonGraphReuseFixtureWithManifest(t *testing.T, sourceManifest interpreter := request.EarlierCatalog[0].Evidence interpreter.RequirementID = "interpreter" interpreter.Terminal.RequirementID = "interpreter" - interpreter.Facts = pythonprovider.CanonicalInterpreterFactsV1("3.13.2") + interpreter.Facts = pythonInterpreterFactsV2ForTest("3.13.2") resolution, err := providers.ResolveProviderNode( context.Background(), request, pythonprovider.WheelNodeResolver{ diff --git a/internal/dockerdeploy/prepared_python_graph_setup.go b/internal/dockerdeploy/prepared_python_graph_setup.go index 1d5168c3..3b532a78 100644 --- a/internal/dockerdeploy/prepared_python_graph_setup.go +++ b/internal/dockerdeploy/prepared_python_graph_setup.go @@ -8,14 +8,16 @@ import ( "github.com/omry/reploy/internal/canonical" "github.com/omry/reploy/internal/deploy" "github.com/omry/reploy/internal/providers" + pythonprovider "github.com/omry/reploy/internal/providers/python" "github.com/omry/reploy/internal/providers/registry" "github.com/omry/reploy/internal/providerstore" ) type PreparedPythonNodeConfig struct { - ReusableWheels []providerstore.ArtifactDescriptor - LocalOverrides []PythonLocalOverrideV1 - SourceBuilder *SourceBuilderCoordinatorV1 + ReusableWheels []providerstore.ArtifactDescriptor + LocalOverrides []PythonLocalOverrideV1 + PortableToolBindings *pythonprovider.PortableToolPythonComponentV1 + SourceBuilder *SourceBuilderCoordinatorV1 } type PreparedAPTNodeConfig struct { @@ -127,6 +129,7 @@ func PreparePreparedPythonGraphBackend( FinalImageConfig: cloneImageConfigPolicy(finalImageConfig), Artifacts: artifacts, ReusableWheels: append([]providerstore.ArtifactDescriptor{}, config.ReusableWheels...), LocalOverrides: append([]PythonLocalOverrideV1{}, config.LocalOverrides...), + PortableToolBindings: config.PortableToolBindings, SourceBuilder: config.SourceBuilder, Progress: options.Progress, ShowApplicationContext: showApplicationContext, diff --git a/internal/dockerdeploy/prepared_python_node_operations.go b/internal/dockerdeploy/prepared_python_node_operations.go index f28d38c6..720ca400 100644 --- a/internal/dockerdeploy/prepared_python_node_operations.go +++ b/internal/dockerdeploy/prepared_python_node_operations.go @@ -29,6 +29,7 @@ type PreparedPythonNodeOperations struct { Artifacts PreparedPythonResolverArtifacts ReusableWheels []providerstore.ArtifactDescriptor LocalOverrides []PythonLocalOverrideV1 + PortableToolBindings *pythonprovider.PortableToolPythonComponentV1 SourceBuilder *SourceBuilderCoordinatorV1 Progress io.Writer ShowApplicationContext bool @@ -128,7 +129,11 @@ func (operations PreparedPythonNodeOperations) validateCached( SupplierNode: providers.NodeID(locked.Output.Component), SupplierComponent: locked.Output.Component, Name: locked.Output.Name, Candidate: providers.ExecutableCandidate{InvocationPath: locked.InvocationPath}, } - observed, err := SelectPythonInterpreter(ctx, session, consumer.EnvironmentLauncher, requirement, []providers.RealizedOutput{candidate}) + testedTags := []string{} + if operations.PortableToolBindings != nil { + testedTags = append(testedTags, operations.PortableToolBindings.TestedTags...) + } + observed, err := SelectPythonInterpreterWithTags(ctx, session, consumer.EnvironmentLauncher, requirement, []providers.RealizedOutput{candidate}, testedTags) if err != nil { return providers.GraphConsumerValidation{}, err } @@ -174,7 +179,11 @@ func (operations PreparedPythonNodeOperations) resolveFresh( } requirement := node.Requirements.Executables[0] candidates := append([]providers.RealizedOutput{}, candidateGroups[0].Outputs...) - interpreter, err := SelectPythonInterpreter(ctx, session, consumer.EnvironmentLauncher, requirement, candidates) + testedTags := []string{} + if operations.PortableToolBindings != nil { + testedTags = append(testedTags, operations.PortableToolBindings.TestedTags...) + } + interpreter, err := SelectPythonInterpreterWithTags(ctx, session, consumer.EnvironmentLauncher, requirement, candidates, testedTags) if err != nil { return providers.ResolveResult{}, providers.GraphConsumerValidation{}, err } @@ -442,6 +451,10 @@ func (operations PreparedPythonNodeOperations) materializeLocalOverrides( var preparePythonSourceBuilderWorkspaceV1 = PrepareProbeWorkspace var openPythonSourceBuilderSessionV1 = OpenPythonResolverSession var validatePythonSourceBuilderConsumerV1 = ValidatePythonConsumer + +// Source-builder inspection deliberately remains tag-less. Portable binding +// tested tags describe runtime-wheel compatibility for the application node; +// they are not compatibility claims for the isolated source-build image. var selectPythonSourceBuilderInterpreterV1 = SelectPythonInterpreter // openSourceBuilderSession prepares and opens the distinct source-build diff --git a/internal/dockerdeploy/prepared_python_node_operations_test.go b/internal/dockerdeploy/prepared_python_node_operations_test.go index ed74bed6..ae1612b3 100644 --- a/internal/dockerdeploy/prepared_python_node_operations_test.go +++ b/internal/dockerdeploy/prepared_python_node_operations_test.go @@ -85,7 +85,8 @@ func TestPreparedPythonNodeOperationsResolvesAndIngestsWheelsInSession(t *testin interpreterObservation := pythonConsumerObservation("interpreter", "/usr/bin/python3") interpreterResponse := probe.ResponseV1{Schema: probe.ResponseSchemaV1, Observations: []probe.ExecutableObservationV1{interpreterObservation}} artifacts := testPreparedPythonResolverArtifacts(t) - commands := stubPythonInterpreterSelectionCommands(t, mustCanonicalProbeResponse(t, interpreterResponse), []string{"3.13.2\n"}, func() error { + testedTags := []string{"py3-none-any"} + commands := stubPythonInterpreterSelectionCommands(t, mustCanonicalProbeResponse(t, interpreterResponse), []string{string(pythonInspectionOutputV2ForTest("3.13.2", testedTags, testedTags))}, func() error { writePythonIntegrationWheel(t, filepath.Join(artifacts.OutputHostDir, "demo_server-1.0-py3-none-any.whl")) return nil }) @@ -108,6 +109,9 @@ func TestPreparedPythonNodeOperationsResolvesAndIngestsWheelsInSession(t *testin }, FinalImageConfig: pythonConsumerTestImageConfig(), Artifacts: artifacts, + PortableToolBindings: &pythonprovider.PortableToolPythonComponentV1{ + TestedTags: testedTags, + }, LocalOverrides: []PythonLocalOverrideV1{{ Distribution: "unused", HostDir: filepath.Join(t.TempDir(), "missing"), }}, @@ -140,10 +144,18 @@ func TestPreparedPythonNodeOperationsResolvesAndIngestsWheelsInSession(t *testin if len(*commands) != 7 { t.Fatalf("commands = %#v", *commands) } + inspection, err := pythonprovider.InterpreterInspectionArgv("/usr/bin/python3", testedTags, "x86_64") + if err != nil { + t.Fatal(err) + } + inspectArgs := (*commands)[3].Args + if len(inspectArgs) < len(inspection) || !reflect.DeepEqual(inspectArgs[len(inspectArgs)-len(inspection):], inspection) { + t.Fatalf("interpreter inspection command = %#v, want suffix %#v", inspectArgs, inspection) + } if !containsInOrder((*commands)[4].Args, []string{ "/usr/bin/env", "-i", "HOME=/tmp", "LANG=C", "LC_ALL=C", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "TMPDIR=/tmp", - "/usr/bin/python3", "-m", "pip", "--disable-pip-version-check", "wheel", + "/usr/bin/python3", "-I", "-m", "pip", "--disable-pip-version-check", "wheel", }) { t.Fatalf("wheel resolver command = %#v", (*commands)[4].Args) } @@ -159,6 +171,49 @@ func TestPreparedPythonNodeOperationsResolvesAndIngestsWheelsInSession(t *testin } } +func TestPreparedPythonNodeOperationsStopsWhenInterpreterCannotImportPip(t *testing.T) { + descriptor := testProbeImageDescriptor(t, "linux/amd64") + workspace := testPreparedProbeWorkspace(t, descriptor.Platform, t.TempDir()) + request := preparedPythonResolveRequest(t, descriptor) + interpreterObservation := pythonConsumerObservation("interpreter", "/usr/bin/python3") + interpreterResponse := probe.ResponseV1{Schema: probe.ResponseSchemaV1, Observations: []probe.ExecutableObservationV1{interpreterObservation}} + resolverCalls := 0 + commands := stubPythonInterpreterSelectionCommands( + t, mustCanonicalProbeResponse(t, interpreterResponse), []string{"error:No module named pip"}, + func() error { resolverCalls++; return nil }, + ) + artifacts := testPreparedPythonResolverArtifacts(t) + session, err := OpenPythonResolverSession(context.Background(), descriptor, workspace, artifacts) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = session.Close(context.Background()) }) + session.observations[pythonCarrierRequirementID] = pythonConsumerObservation(pythonCarrierRequirementID, pythonCarrierPath) + session.observations[pythonLauncherRequirementID] = pythonConsumerObservation(pythonLauncherRequirementID, pythonLauncherPath) + store, err := providerstore.NewStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + operations := PreparedPythonNodeOperations{ + Store: store, Artifacts: artifacts, + Validators: providers.ProviderOwnerValidators{ + Profile: pythonprovider.ValidateRequirementProfileV1, + Bundle: pythonprovider.ValidateResolvedBundlePayloadV1, + }, + FinalImageConfig: pythonConsumerTestImageConfig(), + } + _, _, err = operations.resolveFresh(context.Background(), session, request) + if err == nil || !strings.Contains(err.Error(), "No module named pip") { + t.Fatalf("inspection failure = %v", err) + } + if resolverCalls != 0 { + t.Fatalf("pip resolver ran %d times after interpreter inspection failed", resolverCalls) + } + if len(*commands) != 4 { + t.Fatalf("commands before inspection failure = %#v", *commands) + } +} + func TestPreparedPythonNodeOperationsExcludesCorruptReusableWheelBeforePip(t *testing.T) { descriptor := testProbeImageDescriptor(t, "linux/amd64") workspace := testPreparedProbeWorkspace(t, descriptor.Platform, t.TempDir()) @@ -178,7 +233,7 @@ func TestPreparedPythonNodeOperationsExcludesCorruptReusableWheelBeforePip(t *te t.Fatal(err) } defer cleanup() - commands := stubPythonInterpreterSelectionCommands(t, mustCanonicalProbeResponse(t, interpreterResponse), []string{"3.13.2\n"}, func() error { + commands := stubPythonInterpreterSelectionCommands(t, mustCanonicalProbeResponse(t, interpreterResponse), []string{string(pythonInspectionOutputV2ForTest("3.13.2", nil, nil))}, func() error { writePythonIntegrationWheel(t, filepath.Join(artifacts.OutputHostDir, "demo_server-1.0-py3-none-any.whl")) return nil }) @@ -259,7 +314,7 @@ func TestPreparedPythonNodeOperationsBuildsOnlySelectedLocalSource(t *testing.T) t.Fatal(err) } workCalls := 0 - commands := stubPythonInterpreterSelectionCommands(t, mustCanonicalProbeResponse(t, interpreterResponse), []string{"3.13.2\n"}, func() error { + commands := stubPythonInterpreterSelectionCommands(t, mustCanonicalProbeResponse(t, interpreterResponse), []string{string(pythonInspectionOutputV2ForTest("3.13.2", nil, nil))}, func() error { workCalls++ if workCalls == 1 { writeDockerdeployTestSourceDistribution( diff --git a/internal/dockerdeploy/provider_build_execute.go b/internal/dockerdeploy/provider_build_execute.go index 28648c69..061b1fed 100644 --- a/internal/dockerdeploy/provider_build_execute.go +++ b/internal/dockerdeploy/provider_build_execute.go @@ -263,6 +263,10 @@ func executeLockedProviderBuildV1( graphCtx, endGraph := buildprofile.Start(ctx, "Execute provider graph") graphOptions := options graphOptions.Context = graphCtx + // The ordinary build path deliberately leaves PortablePython unset: its + // request loader rejects unresolved runtime portable tools before provider + // preparation. The generic portable-tool production caller owns supplying + // that projection when it replaces the rejection boundary. graph, err := backend.executeGraph(graphCtx, PreparedPythonGraphExecutionInput{ Store: preparation.Store, Plan: preparedBase.Plan, BaseDescriptor: preparedBase.Descriptor, BaseCatalog: preparedBase.Catalog, Sources: preparation.Loaded.Request.Sources, diff --git a/internal/dockerdeploy/python_inspection_test_helpers_test.go b/internal/dockerdeploy/python_inspection_test_helpers_test.go new file mode 100644 index 00000000..69032785 --- /dev/null +++ b/internal/dockerdeploy/python_inspection_test_helpers_test.go @@ -0,0 +1,49 @@ +package dockerdeploy + +import ( + "testing" + + "github.com/omry/reploy/internal/canonical" + "github.com/omry/reploy/internal/providers" + pythonprovider "github.com/omry/reploy/internal/providers/python" +) + +func pythonInterpreterFactsV2ForTest(version string) providers.CanonicalProviderData { + return pythonprovider.CanonicalInterpreterFactsV2(pythonInspectionFactsV2ForTest(version, nil, nil)) +} + +func pythonInspectionFactsV2ForTest(version string, testedTags, compatibleTags []string) pythonprovider.InterpreterInspectionFactsV2 { + if testedTags == nil { + testedTags = []string{} + } + if compatibleTags == nil { + compatibleTags = []string{} + } + return pythonprovider.InterpreterInspectionFactsV2{ + Version: version, Implementation: "cpython", ABI: "cp313", + Libc: "glibc", LibcMajor: "2", LibcMinor: "35", + TestedTags: append([]string{}, testedTags...), CompatibleTags: append([]string{}, compatibleTags...), + } +} + +func pythonInspectionOutputV2ForTest(version string, testedTags, compatibleTags []string) []byte { + facts := pythonInspectionFactsV2ForTest(version, testedTags, compatibleTags) + encoded, err := canonical.Marshal(facts) + if err != nil { + panic(err) + } + return encoded +} + +func pythonInspectionOutputFromFactsV2ForTest(t *testing.T, data canonical.Envelope) []byte { + t.Helper() + facts, err := pythonprovider.DecodeInterpreterFactsV2(data) + if err != nil { + t.Fatal(err) + } + encoded, err := canonical.Marshal(facts) + if err != nil { + t.Fatal(err) + } + return encoded +} diff --git a/internal/dockerdeploy/python_interpreter_selection.go b/internal/dockerdeploy/python_interpreter_selection.go index 5dfc1e57..33575c7f 100644 --- a/internal/dockerdeploy/python_interpreter_selection.go +++ b/internal/dockerdeploy/python_interpreter_selection.go @@ -18,6 +18,20 @@ func SelectPythonInterpreter( launcher providers.ValidatedExecutableInput, requirement providers.ExecutableRequirement, candidates []providers.RealizedOutput, +) (providers.ExecutableEvidence, error) { + return SelectPythonInterpreterWithTags(ctx, session, launcher, requirement, candidates, []string{}) +} + +// SelectPythonInterpreterWithTags performs the same single interpreter +// selection while asking the fixed inspection to test only the bounded tags +// projected for this Python component. +func SelectPythonInterpreterWithTags( + ctx context.Context, + session *PythonResolverSession, + launcher providers.ValidatedExecutableInput, + requirement providers.ExecutableRequirement, + candidates []providers.RealizedOutput, + testedTags []string, ) (providers.ExecutableEvidence, error) { if ctx == nil { return providers.ExecutableEvidence{}, fmt.Errorf("select Python interpreter requires a context") @@ -63,7 +77,7 @@ func SelectPythonInterpreter( continue } output := providers.QualifiedOutput{Component: candidate.SupplierComponent, Name: candidate.Name} - selected, version, err := session.InspectAndBindInterpreter(ctx, launcher, requirement, output) + selected, facts, err := session.InspectAndBindInterpreter(ctx, launcher, requirement, output, testedTags) if err != nil { candidateErr := fmt.Errorf( "Python interpreter candidate %s.%s at %s is not a usable Python interpreter; configure an explicit Python interpreter executable path: %w", @@ -75,7 +89,7 @@ func SelectPythonInterpreter( lastInvalidCandidate = candidateErr.Error() continue } - matches, err := pythonprovider.InterpreterVersionSatisfies(requirement.VersionConstraint, version) + matches, err := pythonprovider.InterpreterVersionSatisfies(requirement.VersionConstraint, facts.Version) if err != nil { return providers.ExecutableEvidence{}, err } diff --git a/internal/dockerdeploy/python_interpreter_selection_test.go b/internal/dockerdeploy/python_interpreter_selection_test.go index 0155e1f8..bcc561ad 100644 --- a/internal/dockerdeploy/python_interpreter_selection_test.go +++ b/internal/dockerdeploy/python_interpreter_selection_test.go @@ -16,7 +16,10 @@ func TestSelectPythonInterpreterUsesFirstCompatibleObservedRuntime(t *testing.T) workspace := testPreparedProbeWorkspace(t, descriptor.Platform, t.TempDir()) _, exchange := pythonResolverProbeExchange() interpreterResponse := probe.ResponseV1{Schema: probe.ResponseSchemaV1, Observations: []probe.ExecutableObservationV1{exchange.Observations[1]}} - commands := stubPythonInterpreterSelectionCommands(t, mustCanonicalProbeResponse(t, interpreterResponse), []string{"3.10.14\n", "3.13.2\n"}, nil) + commands := stubPythonInterpreterSelectionCommands(t, mustCanonicalProbeResponse(t, interpreterResponse), []string{ + string(pythonInspectionOutputV2ForTest("3.10.14", nil, nil)), + string(pythonInspectionOutputV2ForTest("3.13.2", nil, nil)), + }, nil) session, err := OpenPythonResolverSession(context.Background(), descriptor, workspace, testPreparedPythonResolverArtifacts(t)) if err != nil { t.Fatal(err) @@ -122,8 +125,12 @@ func stubPythonInterpreterSelectionCommands(t *testing.T, probeResponse []byte, if inspectionIndex >= len(inspectionResponses) { return errors.New("unexpected interpreter inspection") } - _, _ = options.Stdout.Write([]byte(inspectionResponses[inspectionIndex])) + response := inspectionResponses[inspectionIndex] inspectionIndex++ + if strings.HasPrefix(response, "error:") { + return errors.New(strings.TrimPrefix(response, "error:")) + } + _, _ = options.Stdout.Write([]byte(response)) return nil }, nil } diff --git a/internal/dockerdeploy/python_resolver_session.go b/internal/dockerdeploy/python_resolver_session.go index 22cc6e8e..c4699d45 100644 --- a/internal/dockerdeploy/python_resolver_session.go +++ b/internal/dockerdeploy/python_resolver_session.go @@ -28,7 +28,7 @@ type PythonResolverSession struct { containerName string runDocker commandRunner observations map[string]probe.ExecutableObservationV1 - inspected map[string]string + inspected map[string]pythonprovider.InterpreterInspectionFactsV2 stopped bool closed bool } @@ -46,10 +46,10 @@ const ( pythonSourceBuildRoot = "/tmp/reploy-source-build" pythonSourceBuilderRoot = "/tmp/reploy-source-builder" pythonSourceUVCacheRoot = "/tmp/reploy-uv-cache" - pythonSourceBuildEnvironmentSchemaV1 = "python-source-build-environment-v1" + pythonSourceBuildEnvironmentSchemaV2 = "python-source-build-environment-v2" ) -type pythonSourceBuildEnvironmentV1 struct { +type pythonSourceBuildEnvironmentV2 struct { Schema string `json:"schema"` Platform blueprint.Platform `json:"platform"` Builder providers.RealizedImageV1 `json:"builder"` @@ -130,7 +130,7 @@ func OpenPythonResolverSession( return &PythonResolverSession{ descriptor: descriptor, upstream: descriptor, workspace: workspace, artifacts: artifacts, containerName: containerName, runDocker: runDocker, observations: map[string]probe.ExecutableObservationV1{}, - inspected: map[string]string{}, + inspected: map[string]pythonprovider.InterpreterInspectionFactsV2{}, }, nil } @@ -252,33 +252,41 @@ func (session *PythonResolverSession) InspectAndBindInterpreter( launcher providers.ValidatedExecutableInput, requirement providers.ExecutableRequirement, output providers.QualifiedOutput, -) (providers.ValidatedExecutableInput, string, error) { + testedTags []string, +) (providers.ValidatedExecutableInput, pythonprovider.InterpreterInspectionFactsV2, error) { + targetArchitecture, err := pythonInspectionArchitectureV2(session.descriptor.Platform) + if err != nil { + return providers.ValidatedExecutableInput{}, pythonprovider.InterpreterInspectionFactsV2{}, err + } inspectionInput, err := session.ValidatedExecutableInput( providers.ExecutableRoleSelectedOutput, requirement, output, providers.CanonicalProviderData{ - Schema: "python-interpreter-inspection-v1", - Value: canonical.Object{"consumer_kind": "python"}, + Schema: "python-interpreter-inspection-v2", + Value: canonical.Object{ + "consumer_kind": "python", "target_architecture": targetArchitecture, + "tested_tags": append([]string{}, testedTags...), + }, }, ) if err != nil { - return providers.ValidatedExecutableInput{}, "", err + return providers.ValidatedExecutableInput{}, pythonprovider.InterpreterInspectionFactsV2{}, err } - version, err := session.InspectInterpreter(ctx, launcher, inspectionInput) + facts, err := session.InspectInterpreter(ctx, launcher, inspectionInput, testedTags) if err != nil { - return providers.ValidatedExecutableInput{}, "", err + return providers.ValidatedExecutableInput{}, pythonprovider.InterpreterInspectionFactsV2{}, err } selected, err := session.ValidatedExecutableInput( providers.ExecutableRoleSelectedOutput, requirement, output, - pythonprovider.CanonicalInterpreterFactsV1(version), + pythonprovider.CanonicalInterpreterFactsV2(facts), ) if err != nil { - return providers.ValidatedExecutableInput{}, "", err + return providers.ValidatedExecutableInput{}, pythonprovider.InterpreterInspectionFactsV2{}, err } - return selected, version, nil + return selected, facts, nil } // InspectInterpreter runs only Python's fixed isolated inspection through an @@ -287,27 +295,28 @@ func (session *PythonResolverSession) InspectInterpreter( ctx context.Context, launcher providers.ValidatedExecutableInput, interpreter providers.ValidatedExecutableInput, -) (string, error) { + testedTags []string, +) (pythonprovider.InterpreterInspectionFactsV2, error) { if session == nil || session.closed || session.stopped { - return "", fmt.Errorf("Python resolver session is not open") + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("Python resolver session is not open") } if ctx == nil { - return "", fmt.Errorf("Python interpreter inspection context is required") + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("Python interpreter inspection context is required") } if err := ctx.Err(); err != nil { - return "", err + return pythonprovider.InterpreterInspectionFactsV2{}, err } if err := providers.ValidateValidatedExecutableInput(launcher); err != nil { - return "", fmt.Errorf("Python resolver environment launcher: %w", err) + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("Python resolver environment launcher: %w", err) } if launcher.Role != providers.ExecutableRoleEnvironmentLauncher { - return "", fmt.Errorf("Python resolver environment launcher role must be %q", providers.ExecutableRoleEnvironmentLauncher) + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("Python resolver environment launcher role must be %q", providers.ExecutableRoleEnvironmentLauncher) } if err := providers.ValidateValidatedExecutableInput(interpreter); err != nil { - return "", fmt.Errorf("Python resolver interpreter: %w", err) + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("Python resolver interpreter: %w", err) } if interpreter.Role != providers.ExecutableRoleSelectedOutput { - return "", fmt.Errorf("Python resolver interpreter role must be %q", providers.ExecutableRoleSelectedOutput) + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("Python resolver interpreter role must be %q", providers.ExecutableRoleSelectedOutput) } for _, required := range []struct { name string @@ -318,7 +327,7 @@ func (session *PythonResolverSession) InspectInterpreter( } { observation, found := session.observations[required.executable.ID] if !found { - return "", fmt.Errorf("Python resolver %s %q was not probed in this container", required.name, required.executable.Evidence.InvocationPath) + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("Python resolver %s %q was not probed in this container", required.name, required.executable.Evidence.InvocationPath) } requirement := providers.ExecutableRequirement{ ID: required.executable.ID, Command: required.executable.Evidence.Output.Name, @@ -328,15 +337,19 @@ func (session *PythonResolverSession) InspectInterpreter( Requirement: &requirement, Output: required.executable.Evidence.Output, Facts: required.executable.Evidence.Facts, }) if err != nil { - return "", fmt.Errorf("Python resolver %s probe evidence: %w", required.name, err) + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("Python resolver %s probe evidence: %w", required.name, err) } if !reflect.DeepEqual(expected, required.executable.Evidence) { - return "", fmt.Errorf("Python resolver %s evidence does not match this container's probe", required.name) + return pythonprovider.InterpreterInspectionFactsV2{}, fmt.Errorf("Python resolver %s evidence does not match this container's probe", required.name) } } - inspection, err := pythonprovider.InterpreterInspectionArgv(interpreter.Evidence.InvocationPath) + targetArchitecture, err := pythonInspectionArchitectureV2(session.descriptor.Platform) if err != nil { - return "", err + return pythonprovider.InterpreterInspectionFactsV2{}, err + } + inspection, err := pythonprovider.InterpreterInspectionArgv(interpreter.Evidence.InvocationPath, testedTags, targetArchitecture) + if err != nil { + return pythonprovider.InterpreterInspectionFactsV2{}, err } args := []string{ "exec", "--user", "0:0", "--workdir", "/", session.containerName, @@ -348,14 +361,33 @@ func (session *PythonResolverSession) InspectInterpreter( var stdout bytes.Buffer var stderr bytes.Buffer if err := session.runDockerCommand(CommandSpec{Name: "docker", Args: args}, RunOptions{Context: ctx, Stdout: &stdout, Stderr: &stderr}); err != nil { - return "", pythonResolverCommandError("inspect interpreter", session.descriptor.Platform.Canonical, stderr.String(), err) + return pythonprovider.InterpreterInspectionFactsV2{}, pythonResolverCommandError("inspect interpreter", session.descriptor.Platform.Canonical, stderr.String(), err) } - version, err := pythonprovider.ParseInterpreterInspectionOutput(stdout.Bytes()) + facts, err := pythonprovider.ParseInterpreterInspectionOutput(stdout.Bytes(), testedTags) if err != nil { + return pythonprovider.InterpreterInspectionFactsV2{}, err + } + session.inspected[interpreter.Evidence.InvocationPath] = facts + return facts, nil +} + +func pythonInspectionArchitectureV2(platform blueprint.Platform) (string, error) { + if err := platform.Validate(); err != nil { return "", err } - session.inspected[interpreter.Evidence.InvocationPath] = version - return version, nil + switch platform.Architecture { + case "amd64": + return "x86_64", nil + case "arm64": + return "aarch64", nil + case "arm": + if platform.Variant == "v7" { + return "armv7l", nil + } + default: + return "", fmt.Errorf("Python interpreter inspection target architecture %q is unsupported", platform.Canonical) + } + return "", fmt.Errorf("Python interpreter inspection target architecture %q is unsupported", platform.Canonical) } // ResolveWheels runs the one provider-owned pip invocation after the selected @@ -522,17 +554,17 @@ func (session *PythonResolverSession) SourceBuildEnvironmentDigest( if session == nil || session.closed || session.stopped { return "", fmt.Errorf("Python resolver session is not open") } - version, inspected := session.inspected[interpreter.InvocationPath] - if !inspected || interpreter.Facts.Schema != pythonprovider.InterpreterFactsSchemaV1 || - interpreter.Facts.Value["version"] != version { + inspectedFacts, inspected := session.inspected[interpreter.InvocationPath] + facts, err := pythonprovider.DecodeInterpreterFactsV2(interpreter.Facts) + if err != nil || !inspected || !reflect.DeepEqual(facts, inspectedFacts) { return "", fmt.Errorf("Python source build environment interpreter was not inspected in this container") } builder, err := realizedImageFromDescriptor(session.descriptor) if err != nil { return "", err } - environment := pythonSourceBuildEnvironmentV1{ - Schema: pythonSourceBuildEnvironmentSchemaV1, + environment := pythonSourceBuildEnvironmentV2{ + Schema: pythonSourceBuildEnvironmentSchemaV2, Platform: session.descriptor.Platform, Builder: builder, Interpreter: interpreter, @@ -541,7 +573,7 @@ func (session *PythonResolverSession) SourceBuildEnvironmentDigest( environment.PortableTools = append([]SourceBuilderPortableToolSelectionV1{}, session.sourceBuilder.Selections...) } return canonical.Sum( - "python-source-build-environment", pythonSourceBuildEnvironmentSchemaV1, environment, + "python-source-build-environment", pythonSourceBuildEnvironmentSchemaV2, environment, ) } @@ -665,8 +697,9 @@ func (session *PythonResolverSession) validateWheelOperationInputs( if !reflect.DeepEqual(expected.Evidence, interpreter) { return fmt.Errorf("Python wheel operation interpreter evidence does not match this container") } - version, inspected := session.inspected[interpreter.InvocationPath] - if !inspected || interpreter.Facts.Schema != pythonprovider.InterpreterFactsSchemaV1 || interpreter.Facts.Value["version"] != version { + inspectedFacts, inspected := session.inspected[interpreter.InvocationPath] + facts, err := pythonprovider.DecodeInterpreterFactsV2(interpreter.Facts) + if err != nil || !inspected || !reflect.DeepEqual(facts, inspectedFacts) { return fmt.Errorf("Python wheel operation interpreter was not inspected in this container") } return nil diff --git a/internal/dockerdeploy/python_resolver_session_test.go b/internal/dockerdeploy/python_resolver_session_test.go index a81862f9..2a48141c 100644 --- a/internal/dockerdeploy/python_resolver_session_test.go +++ b/internal/dockerdeploy/python_resolver_session_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/omry/reploy/internal/blueprint" "github.com/omry/reploy/internal/canonical" "github.com/omry/reploy/internal/probe" "github.com/omry/reploy/internal/providers" @@ -25,7 +26,7 @@ func TestPythonResolverSessionProbesAndInspectsInOneContainer(t *testing.T) { artifacts := testPreparedPythonResolverArtifacts(t) request, responseRecord := pythonResolverProbeExchange() response := mustCanonicalProbeResponse(t, responseRecord) - commands, probeInput := stubPythonResolverCommands(t, response, []byte("3.13.2\n"), nil) + commands, probeInput := stubPythonResolverCommands(t, response, pythonInspectionOutputV2ForTest("3.13.2", nil, nil), nil) session, err := OpenPythonResolverSession(context.Background(), descriptor, workspace, artifacts) if err != nil { @@ -36,12 +37,12 @@ func TestPythonResolverSessionProbesAndInspectsInOneContainer(t *testing.T) { } launcher := pythonResolverSessionInput(t, session, responseRecord.Observations[0], providers.ExecutableRoleEnvironmentLauncher) interpreter := pythonResolverSessionInput(t, session, responseRecord.Observations[1], providers.ExecutableRoleSelectedOutput) - version, err := session.InspectInterpreter(context.Background(), launcher, interpreter) + facts, err := session.InspectInterpreter(context.Background(), launcher, interpreter, []string{}) if err != nil { t.Fatal(err) } - if version != "3.13.2" { - t.Fatalf("version = %q", version) + if facts.Version != "3.13.2" { + t.Fatalf("facts = %#v", facts) } if err := session.Close(context.Background()); err != nil { t.Fatal(err) @@ -58,12 +59,16 @@ func TestPythonResolverSessionProbesAndInspectsInOneContainer(t *testing.T) { "--mount", "type=bind,source=" + artifacts.OutputHostDir + ",target=" + pythonResolverOutputContainerDir, "--entrypoint", ProbeContainerExecutable, string(descriptor.ConfigDigest), "hold", } + inspection, err := pythonprovider.InterpreterInspectionArgv("/usr/bin/python3", []string{}, "x86_64") + if err != nil { + t.Fatal(err) + } wantInspect := []string{ "exec", "--user", "0:0", "--workdir", "/", name, "/usr/bin/env", "-i", "HOME=/tmp", "LANG=C", "LC_ALL=C", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "TMPDIR=/tmp", - "/usr/bin/python3", "-I", "-S", "-c", `import sys; print(".".join(map(str, sys.version_info[:3])))`, } + wantInspect = append(wantInspect, inspection...) want := [][]string{ wantCreate, {"start", name}, @@ -84,6 +89,29 @@ func TestPythonResolverSessionProbesAndInspectsInOneContainer(t *testing.T) { } } +func TestPythonInspectionArchitecturePreservesSupportedProviderPlatforms(t *testing.T) { + for platform, want := range map[string]string{ + "linux/amd64": "x86_64", + "linux/arm64": "aarch64", + "linux/arm64/v8": "aarch64", + "linux/arm/v7": "armv7l", + } { + t.Run(platform, func(t *testing.T) { + parsed, err := blueprint.ParsePlatform(platform) + if err != nil { + t.Fatal(err) + } + got, err := pythonInspectionArchitectureV2(parsed) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("inspection architecture = %q, want %q", got, want) + } + }) + } +} + func TestPythonResolverSessionRejectsWrongTypedRolesBeforeExec(t *testing.T) { descriptor := testProbeImageDescriptor(t, "linux/amd64") workspace := testPreparedProbeWorkspace(t, descriptor.Platform, t.TempDir()) @@ -94,7 +122,7 @@ func TestPythonResolverSessionRejectsWrongTypedRolesBeforeExec(t *testing.T) { } launcher := rendererExecutable("cleanenv", providers.ExecutableRoleCarrier, "/usr/bin/env") interpreter := rendererExecutable("interpreter", providers.ExecutableRoleSelectedOutput, "/usr/bin/python3") - if _, err := session.InspectInterpreter(context.Background(), launcher, interpreter); err == nil || !strings.Contains(err.Error(), "environment-launcher") { + if _, err := session.InspectInterpreter(context.Background(), launcher, interpreter, []string{}); err == nil || !strings.Contains(err.Error(), "environment-launcher") { t.Fatalf("wrong launcher error = %v", err) } if len(*commands) != 2 { @@ -115,7 +143,7 @@ func TestPythonResolverSessionRequiresSameContainerProbeBeforeInspection(t *test } launcher := rendererExecutable("cleanenv", providers.ExecutableRoleEnvironmentLauncher, "/usr/bin/env") interpreter := rendererExecutable("interpreter", providers.ExecutableRoleSelectedOutput, "/usr/bin/python3") - if _, err := session.InspectInterpreter(context.Background(), launcher, interpreter); err == nil || !strings.Contains(err.Error(), "was not probed in this container") { + if _, err := session.InspectInterpreter(context.Background(), launcher, interpreter, []string{}); err == nil || !strings.Contains(err.Error(), "was not probed in this container") { t.Fatalf("unprobed inspection error = %v", err) } if len(*commands) != 2 { @@ -225,7 +253,7 @@ func TestPythonResolverSessionBuildsSdistThenWheelWithSelectedInterpreterAndPinn workspace := testPreparedProbeWorkspace(t, descriptor.Platform, t.TempDir()) artifacts := testPreparedPythonResolverArtifacts(t) request, responseRecord := pythonResolverProbeExchange() - commands, _ := stubPythonResolverCommands(t, mustCanonicalProbeResponse(t, responseRecord), []byte("3.13.2\n"), nil) + commands, _ := stubPythonResolverCommands(t, mustCanonicalProbeResponse(t, responseRecord), pythonInspectionOutputV2ForTest("3.13.2", nil, nil), nil) session, err := OpenPythonResolverSession(context.Background(), descriptor, workspace, artifacts) if err != nil { t.Fatal(err) @@ -239,6 +267,7 @@ func TestPythonResolverSessionBuildsSdistThenWheelWithSelectedInterpreterAndPinn } interpreter, _, err := session.InspectAndBindInterpreter( context.Background(), launcher, requirement, providers.QualifiedOutput{Component: "base", Name: "interpreter"}, + []string{}, ) if err != nil { t.Fatal(err) diff --git a/internal/dockerdeploy/source_builder_consumer_test.go b/internal/dockerdeploy/source_builder_consumer_test.go index a8947c32..4c57115a 100644 --- a/internal/dockerdeploy/source_builder_consumer_test.go +++ b/internal/dockerdeploy/source_builder_consumer_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/omry/reploy/internal/blueprint" - "github.com/omry/reploy/internal/canonical" "github.com/omry/reploy/internal/deploy" "github.com/omry/reploy/internal/providers" pythonprovider "github.com/omry/reploy/internal/providers/python" @@ -32,20 +31,20 @@ func sourceBuilderTestEnvironment(t *testing.T, builder deploy.ImageDescriptor, func sourceBuilderTestInterpreter() providers.ExecutableEvidence { return providers.ExecutableEvidence{ InvocationPath: "/usr/bin/python3", - Facts: providers.CanonicalProviderData{Schema: pythonprovider.InterpreterFactsSchemaV1, Value: canonical.Object{"version": "3.13.2"}}, + Facts: pythonInterpreterFactsV2ForTest("3.13.2"), } } func TestPythonResolverSessionSourceBuildEnvironmentIdentityBindsExactBuilderAndSelections(t *testing.T) { upstream := sourceBuilderTestImageDescriptor(t, "1") - plain := &PythonResolverSession{descriptor: upstream, upstream: upstream, inspected: map[string]string{"/usr/bin/python3": "3.13.2"}} + plain := &PythonResolverSession{descriptor: upstream, upstream: upstream, inspected: map[string]pythonprovider.InterpreterInspectionFactsV2{"/usr/bin/python3": pythonInspectionFactsV2ForTest("3.13.2", nil, nil)}} interpreter := sourceBuilderTestInterpreter() plainDigest, err := plain.SourceBuildEnvironmentDigest(interpreter) if err != nil { t.Fatal(err) } firstBuilder := sourceBuilderTestImageDescriptor(t, "2") - first := &PythonResolverSession{descriptor: firstBuilder, upstream: firstBuilder, containerName: "first", inspected: map[string]string{"/usr/bin/python3": "3.13.2"}} + first := &PythonResolverSession{descriptor: firstBuilder, upstream: firstBuilder, containerName: "first", inspected: map[string]pythonprovider.InterpreterInspectionFactsV2{"/usr/bin/python3": pythonInspectionFactsV2ForTest("3.13.2", nil, nil)}} if err := first.BindSourceBuilder(sourceBuilderTestEnvironment(t, firstBuilder, upstream)); err != nil { t.Fatal(err) } @@ -54,7 +53,7 @@ func TestPythonResolverSessionSourceBuildEnvironmentIdentityBindsExactBuilderAnd t.Fatal(err) } secondBuilder := sourceBuilderTestImageDescriptor(t, "3") - second := &PythonResolverSession{descriptor: secondBuilder, upstream: secondBuilder, containerName: "second", inspected: map[string]string{"/usr/bin/python3": "3.13.2"}} + second := &PythonResolverSession{descriptor: secondBuilder, upstream: secondBuilder, containerName: "second", inspected: map[string]pythonprovider.InterpreterInspectionFactsV2{"/usr/bin/python3": pythonInspectionFactsV2ForTest("3.13.2", nil, nil)}} if err := second.BindSourceBuilder(sourceBuilderTestEnvironment(t, secondBuilder, upstream)); err != nil { t.Fatal(err) } @@ -86,6 +85,26 @@ func TestPythonResolverSessionSourceBuildEnvironmentIdentityBindsExactBuilderAnd } } +func TestPythonResolverSessionRejectsConflictingFactsForInspectedInterpreterPath(t *testing.T) { + upstream := sourceBuilderTestImageDescriptor(t, "1") + inspected := pythonInspectionFactsV2ForTest("3.13.2", []string{"py3-none-any"}, []string{"py3-none-any"}) + session := &PythonResolverSession{ + descriptor: upstream, + upstream: upstream, + inspected: map[string]pythonprovider.InterpreterInspectionFactsV2{ + "/usr/bin/python3": inspected, + }, + } + interpreter := sourceBuilderTestInterpreter() + interpreter.Facts = pythonprovider.CanonicalInterpreterFactsV2( + pythonInspectionFactsV2ForTest("3.13.2", []string{"py3-none-any"}, []string{}), + ) + if _, err := session.SourceBuildEnvironmentDigest(interpreter); err == nil || + !strings.Contains(err.Error(), "interpreter was not inspected in this container") { + t.Fatalf("conflicting interpreter facts error = %v", err) + } +} + func TestPythonResolverSessionExposesSelectedExportsOnlyToSourceBuilds(t *testing.T) { builder := sourceBuilderTestImageDescriptor(t, "2") commands := []CommandSpec{} diff --git a/internal/providers/python/component_provider.go b/internal/providers/python/component_provider.go index ca47b3dd..e54773f4 100644 --- a/internal/providers/python/component_provider.go +++ b/internal/providers/python/component_provider.go @@ -10,8 +10,8 @@ import ( ) const ( - RecipeVersion = "python-v1" - MaterializationRecipeVersion = "python-materialize-v1" + RecipeVersion = "python-v2" + MaterializationRecipeVersion = "python-materialize-v2" InstallRoot = "/opt/reploy/providers/python" BundleMount = "/reploy-bundle" ) diff --git a/internal/providers/python/interpreter_inspection.go b/internal/providers/python/interpreter_inspection.go index 1a5529c2..714492dd 100644 --- a/internal/providers/python/interpreter_inspection.go +++ b/internal/providers/python/interpreter_inspection.go @@ -1,41 +1,245 @@ package python import ( + "bytes" + "encoding/json" "fmt" + "io" "path" "strconv" "strings" + + "github.com/omry/reploy/internal/portabletool" ) -const interpreterInspectionProgram = `import sys; print(".".join(map(str, sys.version_info[:3])))` +const ( + InterpreterFactsSchemaV2 = "python-interpreter-facts-v2" + interpreterInspectionProgramV2 = `import json,platform,sys +from pip._vendor.packaging import tags +tested=json.loads(sys.argv[1]) +target=sys.argv[2] +if target not in ("x86_64","aarch64","armv7l"): raise RuntimeError("unsupported target architecture") +all_tags=list(tags.sys_tags()) +if not all_tags: raise RuntimeError("pip returned no interpreter tags") +architecture_aliases={"x86_64":("x86_64",),"aarch64":("aarch64","arm64"),"armv7l":("armv7l",)} +if not any(item.platform != "any" and any(item.platform.endswith("_"+architecture) for architecture in architecture_aliases[target]) for item in all_tags): raise RuntimeError("interpreter architecture does not match selected target") +tested_set=set(tested) +compatible=sorted({str(item) for item in all_tags if str(item) in tested_set}) +implementation=sys.implementation.name.lower() +interpreter=tags.interpreter_name()+tags.interpreter_version() +abi=next((item.abi for item in all_tags if item.interpreter == interpreter), "none") +libc,release=platform.libc_ver() +libc=libc.lower() or "unknown" +parts=release.split(".") if release else [] +major=parts[0] if parts and parts[0].isdigit() else "0" +minor=parts[1] if len(parts)>1 and parts[1].isdigit() else "0" +version=".".join(map(str,sys.version_info[:3])) +print(json.dumps({"version":version,"implementation":implementation,"abi":abi,"libc":libc,"libc_major":major,"libc_minor":minor,"tested_tags":sorted(tested),"compatible_tags":compatible},separators=(",",":")))` +) -// InterpreterInspectionArgv returns the complete fixed invocation used by a -// Python consumer to identify the selected absolute interpreter. The caller -// executes this argv inside its existing resolver or materializer container. -func InterpreterInspectionArgv(executable string) ([]string, error) { +// IsolatedInterpreterCommandPrefixV2 is the one command prefix shared by +// interpreter inspection and ordinary pip resolution. +func IsolatedInterpreterCommandPrefixV2(executable string) ([]string, error) { if executable == "" || !path.IsAbs(executable) || path.Clean(executable) != executable || strings.Contains(executable, `\`) { return nil, fmt.Errorf("Python interpreter executable %q must be a normalized absolute Linux path", executable) } - return []string{executable, "-I", "-S", "-c", interpreterInspectionProgram}, nil + return []string{executable, "-I"}, nil +} + +// InterpreterInspectionArgv returns the complete fixed V2 invocation used by +// a Python consumer to inspect the selected absolute interpreter. The tested +// tags and target architecture are validated argv data, never Python source. +func InterpreterInspectionArgv(executable string, testedTags []string, targetArchitecture string) ([]string, error) { + prefix, err := IsolatedInterpreterCommandPrefixV2(executable) + if err != nil { + return nil, err + } + if err := validateInspectionTagsV2(testedTags); err != nil { + return nil, fmt.Errorf("Python inspection tested tags: %w", err) + } + if targetArchitecture != "x86_64" && targetArchitecture != "aarch64" && targetArchitecture != "armv7l" { + return nil, fmt.Errorf("Python inspection target architecture %q is unsupported", targetArchitecture) + } + tagJSON, err := json.Marshal(testedTags) + if err != nil { + return nil, fmt.Errorf("encode Python inspection tested tags: %w", err) + } + return append(prefix, "-c", interpreterInspectionProgramV2, string(tagJSON), targetArchitecture), nil +} + +// InterpreterInspectionFactsV2 is the strict wire representation emitted by +// the fixed probe. Decimal release components are strings because canonical +// provider data does not contain numeric values. +type InterpreterInspectionFactsV2 struct { + Version string `json:"version"` + Implementation string `json:"implementation"` + ABI string `json:"abi"` + Libc string `json:"libc"` + LibcMajor string `json:"libc_major"` + LibcMinor string `json:"libc_minor"` + TestedTags []string `json:"tested_tags"` + CompatibleTags []string `json:"compatible_tags"` +} + +// ParseInterpreterInspectionOutput strictly parses one complete probe JSON +// document and requires its tested set to equal the validated request. +func ParseInterpreterInspectionOutput(output []byte, expectedTestedTags []string) (InterpreterInspectionFactsV2, error) { + if err := validateInspectionTagsV2(expectedTestedTags); err != nil { + return InterpreterInspectionFactsV2{}, fmt.Errorf("expected tested tags: %w", err) + } + value, err := decodeInspectionObjectV2(output) + if err != nil { + return InterpreterInspectionFactsV2{}, err + } + encoded, err := json.Marshal(value) + if err != nil { + return InterpreterInspectionFactsV2{}, err + } + var facts InterpreterInspectionFactsV2 + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&facts); err != nil { + return InterpreterInspectionFactsV2{}, fmt.Errorf("decode Python interpreter inspection: %w", err) + } + if err := validateInspectionFactsV2(facts, expectedTestedTags); err != nil { + return InterpreterInspectionFactsV2{}, err + } + return facts, nil } -// ParseInterpreterInspectionOutput accepts only the three canonical numeric -// components emitted by interpreterInspectionProgram. Diagnostics and other -// output cannot be mistaken for a Python version. -func ParseInterpreterInspectionOutput(output []byte) (string, error) { - value := string(output) - if strings.HasSuffix(value, "\n") { - value = strings.TrimSuffix(value, "\n") +func decodeInspectionObjectV2(output []byte) (map[string]json.RawMessage, error) { + decoder := json.NewDecoder(bytes.NewReader(output)) + token, err := decoder.Token() + if err != nil { + return nil, fmt.Errorf("decode Python interpreter inspection: %w", err) + } + if token != json.Delim('{') { + return nil, fmt.Errorf("Python interpreter inspection must be a JSON object") + } + value := make(map[string]json.RawMessage) + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return nil, fmt.Errorf("decode Python interpreter inspection key: %w", err) + } + key, ok := keyToken.(string) + if !ok { + return nil, fmt.Errorf("Python interpreter inspection object key is not a string") + } + if _, exists := value[key]; exists { + return nil, fmt.Errorf("Python interpreter inspection contains duplicate field %q", key) + } + var raw json.RawMessage + if err := decoder.Decode(&raw); err != nil { + return nil, fmt.Errorf("decode Python interpreter inspection field %q: %w", key, err) + } + value[key] = raw } - parts := strings.Split(value, ".") - if len(parts) != 3 { - return "", fmt.Errorf("Python interpreter inspection returned %q, want a three-part release version", value) + if token, err = decoder.Token(); err != nil || token != json.Delim('}') { + return nil, fmt.Errorf("Python interpreter inspection has an unterminated object") } - for _, part := range parts { - parsed, err := strconv.Atoi(part) - if err != nil || parsed < 0 || strconv.Itoa(parsed) != part { - return "", fmt.Errorf("Python interpreter inspection returned %q, want a canonical three-part release version", value) + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + if err == nil { + return nil, fmt.Errorf("Python interpreter inspection contains trailing JSON") } + return nil, fmt.Errorf("Python interpreter inspection contains trailing data: %w", err) } return value, nil } + +func validateInspectionFactsV2(facts InterpreterInspectionFactsV2, expected []string) error { + if facts.Version == "" || ValidateInterpreterVersionV1(facts.Version) != nil || len(strings.Split(facts.Version, ".")) != 3 { + return fmt.Errorf("Python interpreter inspection version %q is not canonical", facts.Version) + } + if !canonicalInspectionComponentV2(facts.Implementation, false) || !canonicalInspectionComponentV2(facts.ABI, true) { + return fmt.Errorf("Python interpreter inspection implementation or ABI is not canonical") + } + if !canonicalInspectionComponentV2(facts.Libc, false) { + return fmt.Errorf("Python interpreter inspection libc is not canonical") + } + if err := validateInspectionDecimalV2(facts.LibcMajor); err != nil { + return fmt.Errorf("Python interpreter inspection libc major: %w", err) + } + if err := validateInspectionDecimalV2(facts.LibcMinor); err != nil { + return fmt.Errorf("Python interpreter inspection libc minor: %w", err) + } + if err := validateInspectionTagsV2(facts.TestedTags); err != nil { + return fmt.Errorf("Python interpreter inspection tested tags: %w", err) + } + if err := validateInspectionTagsV2(facts.CompatibleTags); err != nil { + return fmt.Errorf("Python interpreter inspection compatible tags: %w", err) + } + if !equalInspectionStringsV2(facts.TestedTags, expected) { + return fmt.Errorf("Python interpreter inspection tested tags do not equal requested tags") + } + tested := make(map[string]struct{}, len(facts.TestedTags)) + for _, tag := range facts.TestedTags { + tested[tag] = struct{}{} + } + for _, tag := range facts.CompatibleTags { + if _, ok := tested[tag]; !ok { + return fmt.Errorf("Python interpreter inspection compatible tag %q is outside tested tags", tag) + } + } + return nil +} + +func validateInspectionTagsV2(tags []string) error { + if tags == nil { + return fmt.Errorf("tags must use a canonical array") + } + if len(tags) > portabletool.RecordArrayMaxEntriesV1 { + return fmt.Errorf("tags exceed the bounded record limit of %d", portabletool.RecordArrayMaxEntriesV1) + } + for index, tag := range tags { + parts := strings.Split(tag, "-") + if len(parts) != 3 || !inspectionTagComponentV2(parts[0]) || !inspectionTagComponentV2(parts[1]) || !inspectionTagComponentV2(parts[2]) { + return fmt.Errorf("tag %q is not a canonical three-part tag", tag) + } + if index > 0 && tags[index-1] >= tag { + return fmt.Errorf("tags must be unique and sorted") + } + } + return nil +} + +func inspectionTagComponentV2(value string) bool { + if value == "" { + return false + } + for index := 0; index < len(value); index++ { + character := value[index] + if character != '_' && (character < 'a' || character > 'z') && (character < '0' || character > '9') { + return false + } + } + return true +} + +func canonicalInspectionComponentV2(value string, allowNone bool) bool { + return allowNone && value == "none" || inspectionTagComponentV2(value) +} + +func validateInspectionDecimalV2(value string) error { + if value == "" || len(value) > 1 && value[0] == '0' { + return fmt.Errorf("must be a canonical nonnegative decimal") + } + parsed, err := strconv.ParseUint(value, 10, 31) + if err != nil || parsed > 1<<31-1 { + return fmt.Errorf("must be a bounded nonnegative decimal") + } + return nil +} + +func equalInspectionStringsV2(left, right []string) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} diff --git a/internal/providers/python/interpreter_inspection_test.go b/internal/providers/python/interpreter_inspection_test.go index 8f068a75..45fef934 100644 --- a/internal/providers/python/interpreter_inspection_test.go +++ b/internal/providers/python/interpreter_inspection_test.go @@ -1,47 +1,302 @@ package python import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" "reflect" + "runtime" "strings" "testing" ) func TestInterpreterInspectionArgvIsFixedAndAbsolute(t *testing.T) { - argv, err := InterpreterInspectionArgv("/usr/bin/python3") + tested := []string{"cp313-cp313-manylinux2014_x86_64", "py3-none-any"} + argv, err := InterpreterInspectionArgv("/usr/bin/python3", tested, "x86_64") if err != nil { t.Fatal(err) } - want := []string{ - "/usr/bin/python3", "-I", "-S", "-c", - `import sys; print(".".join(map(str, sys.version_info[:3])))`, - } + want := []string{"/usr/bin/python3", "-I", "-c", interpreterInspectionProgramV2, `["cp313-cp313-manylinux2014_x86_64","py3-none-any"]`, "x86_64"} if !reflect.DeepEqual(argv, want) { t.Fatalf("inspection argv = %#v, want %#v", argv, want) } for _, executable := range []string{"", "python3", "/usr/bin/../bin/python3", `/usr/bin\python3`} { - if _, err := InterpreterInspectionArgv(executable); err == nil { + if _, err := InterpreterInspectionArgv(executable, tested, "x86_64"); err == nil { t.Fatalf("inspection accepted executable %q", executable) } } + for _, invalid := range []struct { + tags []string + architecture string + }{ + {nil, "x86_64"}, + {[]string{"py3-none-any", "py3-none-any"}, "x86_64"}, + {[]string{"py3-none-any"}, "amd64"}, + } { + if _, err := InterpreterInspectionArgv("/usr/bin/python3", invalid.tags, invalid.architecture); err == nil { + t.Fatalf("inspection accepted tags %#v and architecture %q", invalid.tags, invalid.architecture) + } + } + if !strings.Contains(argv[3], "sys.argv[1]") || strings.Contains(argv[3], tested[0]) { + t.Fatalf("tested tags were interpolated into fixed source: %#v", argv) + } } func TestParseInterpreterInspectionOutput(t *testing.T) { - for _, output := range []string{"3.13.2\n", "3.13.2"} { - version, err := ParseInterpreterInspectionOutput([]byte(output)) + tested := []string{"py3-none-any"} + valid := `{"version":"3.13.2","implementation":"cpython","abi":"cp313","libc":"glibc","libc_major":"2","libc_minor":"35","tested_tags":["py3-none-any"],"compatible_tags":["py3-none-any"]}` + for _, output := range []string{valid + "\n", valid} { + facts, err := ParseInterpreterInspectionOutput([]byte(output), tested) if err != nil { t.Fatal(err) } - if version != "3.13.2" { - t.Fatalf("version = %q", version) + if facts.Version != "3.13.2" || !reflect.DeepEqual(facts.TestedTags, tested) || !reflect.DeepEqual(facts.CompatibleTags, tested) { + t.Fatalf("facts = %#v", facts) } } for _, output := range []string{ - "", "3.13", "3.13.2.1", "3.013.2", "3.13.x", " 3.13.2\n", "3.13.2\nwarning\n", + "", "3.13.2", "[]", valid + " {}", + `{"version":"3.13.2","implementation":"cpython","abi":"cp313","libc":"glibc","libc_major":"2","libc_minor":"35","tested_tags":null,"compatible_tags":[]}`, + `{"version":"3.13.2","implementation":"cpython","abi":"cp313","libc":"glibc","libc_major":"2","libc_minor":"35","tested_tags":["py3-none-any"],"compatible_tags":[],"extra":true}`, } { t.Run(strings.ReplaceAll(output, "\n", "_"), func(t *testing.T) { - if _, err := ParseInterpreterInspectionOutput([]byte(output)); err == nil { + if _, err := ParseInterpreterInspectionOutput([]byte(output), tested); err == nil { t.Fatalf("inspection accepted output %q", output) } }) } } + +func TestParseInterpreterInspectionOutputRejectsIncompleteOrDishonestTagEvidence(t *testing.T) { + tested := []string{"cp313-cp313-manylinux2014_x86_64", "py3-none-any"} + for _, output := range []string{ + `{"version":"3.13.2","implementation":"cpython","abi":"cp313","libc":"glibc","libc_major":"2","libc_minor":"35","tested_tags":["py3-none-any"],"compatible_tags":["py3-none-any"]}`, + `{"version":"3.13.2","implementation":"cpython","abi":"cp313","libc":"glibc","libc_major":"2","libc_minor":"35","tested_tags":["cp313-cp313-manylinux2014_x86_64","py3-none-any"],"compatible_tags":["outside-none-any"]}`, + `{"version":"3.13.2","implementation":"cpython","abi":"cp313","libc":"glibc","libc_major":"2","libc_minor":"35","tested_tags":["cp313-cp313-manylinux2014_x86_64","py3-none-any"],"compatible_tags":["py3-none-any","py3-none-any"]}`, + `{"version":"3.13.2","version":"3.13.3","implementation":"cpython","abi":"cp313","libc":"glibc","libc_major":"2","libc_minor":"35","tested_tags":["cp313-cp313-manylinux2014_x86_64","py3-none-any"],"compatible_tags":[]}`, + } { + if _, err := ParseInterpreterInspectionOutput([]byte(output), tested); err == nil { + t.Errorf("inspection accepted invalid evidence %s", output) + } + } +} + +func TestParseInterpreterInspectionOutputRejectsMalformedScalarFacts(t *testing.T) { + valid := `{"version":"3.13.2","implementation":"cpython","abi":"cp313","libc":"glibc","libc_major":"2","libc_minor":"35","tested_tags":[],"compatible_tags":[]}` + for _, test := range []struct { + name string + oldValue string + invalidValue string + }{ + {name: "incomplete version", oldValue: `"version":"3.13.2"`, invalidValue: `"version":"3.13"`}, + {name: "implementation", oldValue: `"implementation":"cpython"`, invalidValue: `"implementation":"CPython"`}, + {name: "ABI", oldValue: `"abi":"cp313"`, invalidValue: `"abi":"cp313-d"`}, + {name: "libc", oldValue: `"libc":"glibc"`, invalidValue: `"libc":"glibc!"`}, + {name: "leading-zero major", oldValue: `"libc_major":"2"`, invalidValue: `"libc_major":"02"`}, + {name: "negative minor", oldValue: `"libc_minor":"35"`, invalidValue: `"libc_minor":"-1"`}, + } { + t.Run(test.name, func(t *testing.T) { + output := strings.Replace(valid, test.oldValue, test.invalidValue, 1) + if _, err := ParseInterpreterInspectionOutput([]byte(output), []string{}); err == nil { + t.Fatalf("inspection accepted malformed %s: %s", test.name, output) + } + }) + } +} + +func TestInterpreterInspectionAndResolverShareInstalledPipIsolation(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("live isolation probe requires a host Python at a Linux-style absolute path") + } + python := isolatedPythonForTest(t) + targetArchitecture := "x86_64" + otherArchitecture := "aarch64" + if runtime.GOARCH == "arm64" { + targetArchitecture, otherArchitecture = "aarch64", "x86_64" + } else if runtime.GOARCH != "amd64" { + t.Fatalf("pip inspection test does not support host architecture %q", runtime.GOARCH) + } + shadow := t.TempDir() + if err := os.Mkdir(filepath.Join(shadow, "pip"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(shadow, "pip", "__init__.py"), []byte(`raise RuntimeError("current-directory pip shadow imported")`), 0o644); err != nil { + t.Fatal(err) + } + argv, err := InterpreterInspectionArgv(python, []string{"py3-none-any"}, targetArchitecture) + if err != nil { + t.Fatal(err) + } + command := exec.Command(argv[0], argv[1:]...) + command.Dir = shadow + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("isolated inspection imported a shadow or failed: %v: %s", err, output) + } + facts, err := ParseInterpreterInspectionOutput(output, []string{"py3-none-any"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(facts.CompatibleTags, []string{"py3-none-any"}) { + t.Fatalf("compatible tags = %#v", facts.CompatibleTags) + } + mismatchArgv, err := InterpreterInspectionArgv(python, []string{"py3-none-any"}, otherArchitecture) + if err != nil { + t.Fatal(err) + } + mismatch := exec.Command(mismatchArgv[0], mismatchArgv[1:]...) + mismatch.Dir = shadow + if output, err := mismatch.CombinedOutput(); err == nil || !strings.Contains(string(output), "architecture does not match selected target") { + t.Fatalf("architecture mismatch error = %v: %s", err, output) + } + prefix, err := IsolatedInterpreterCommandPrefixV2(python) + if err != nil { + t.Fatal(err) + } + resolver := exec.Command(prefix[0], append(prefix[1:], "-m", "pip", "--version")...) + resolver.Dir = shadow + resolverOutput, err := resolver.CombinedOutput() + if err != nil || !strings.Contains(strings.ToLower(string(resolverOutput)), "pip") { + t.Fatalf("isolated resolver did not execute installed pip: %v: %s", err, resolverOutput) + } +} + +func TestInterpreterInspectionUsesGeneratedTagArchitecture(t *testing.T) { + python := isolatedPythonForTest(t) + run := func(t *testing.T, platformTag, target string) ([]byte, error) { + t.Helper() + const generatedTags = "all_tags=list(tags.sys_tags())" + replacement := `all_tags=[tags.Tag("cp313","cp313","` + platformTag + `"),tags.Tag("py3","none","any")]` + program := strings.Replace(interpreterInspectionProgramV2, generatedTags, replacement, 1) + if program == interpreterInspectionProgramV2 { + t.Fatal("inspection fixture did not replace generated tags") + } + return exec.Command(python, "-I", "-c", program, "[]", target).CombinedOutput() + } + for _, test := range []struct { + name string + platformTag string + target string + }{ + {name: "amd64", platformTag: "linux_x86_64", target: "x86_64"}, + {name: "arm64", platformTag: "linux_aarch64", target: "aarch64"}, + {name: "armv7", platformTag: "linux_armv7l", target: "armv7l"}, + } { + t.Run(test.name, func(t *testing.T) { + output, err := run(t, test.platformTag, test.target) + if err != nil { + t.Fatalf("matching generated architecture failed: %v: %s", err, output) + } + if _, err := ParseInterpreterInspectionOutput(output, []string{}); err != nil { + t.Fatalf("parse matching generated architecture: %v: %s", err, output) + } + }) + } + + target, wrongPlatform := "x86_64", "linux_i686" + if runtime.GOARCH == "arm64" { + target, wrongPlatform = "aarch64", "linux_x86_64" + } else if runtime.GOARCH != "amd64" { + t.Skipf("wrong-bitness regression does not support host architecture %q", runtime.GOARCH) + } + output, err := run(t, wrongPlatform, target) + if err == nil || !strings.Contains(string(output), "architecture does not match selected target") { + t.Fatalf("generated architecture mismatch error = %v: %s", err, output) + } +} + +func testInterpreterFactsV2(version string, testedTags, compatibleTags []string) InterpreterInspectionFactsV2 { + if testedTags == nil { + testedTags = []string{} + } + if compatibleTags == nil { + compatibleTags = []string{} + } + return InterpreterInspectionFactsV2{ + Version: version, Implementation: "cpython", ABI: "cp313", + Libc: "glibc", LibcMajor: "2", LibcMinor: "35", + TestedTags: append([]string{}, testedTags...), CompatibleTags: append([]string{}, compatibleTags...), + } +} + +func isolatedPythonForTest(t *testing.T) string { + t.Helper() + executable, err := exec.LookPath("python3") + if err != nil { + t.Fatal("python3 is required for pip inspection tests") + } + executable, err = filepath.Abs(executable) + if err != nil { + t.Fatal(err) + } + return filepath.Clean(executable) +} + +func TestSelectedPipManylinuxPolicyHooksRemainAuthoritative(t *testing.T) { + python := isolatedPythonForTest(t) + program := `import inspect,json,sys,types +from pip._vendor.packaging import _manylinux +_manylinux._have_compatible_abi=lambda *args:True +def generated(arch,release,module): + _manylinux._get_glibc_version=lambda:release + if module is None: sys.modules.pop("_manylinux",None) + else: sys.modules["_manylinux"]=module + parameters=inspect.signature(_manylinux.platform_tags).parameters + if len(parameters)==1: return set(_manylinux.platform_tags([arch])) + return set(_manylinux.platform_tags("linux_"+arch,arch)) +def module(mode): + if mode=="absent": return None + if mode=="true": return types.SimpleNamespace(manylinux_compatible=lambda *args:True) + if mode=="false": return types.SimpleNamespace(manylinux_compatible=lambda *args:False) + if mode=="none": return types.SimpleNamespace(manylinux_compatible=lambda *args:None) + if mode=="non_monotonic": return types.SimpleNamespace(manylinux_compatible=lambda major,minor,arch:minor!=16) + if mode=="raising": return types.SimpleNamespace(manylinux_compatible=lambda *args:(_ for _ in ()).throw(RuntimeError("hook raised"))) + attribute,value=mode.rsplit("_",1) + return types.SimpleNamespace(**{attribute:value=="true"}) +cases=[ + ("absent","x86_64",(2,17),"absent","manylinux_2_17_x86_64"), + ("true","x86_64",(2,17),"true","manylinux_2_17_x86_64"), + ("false","x86_64",(2,17),"false","manylinux_2_17_x86_64"), + ("none","x86_64",(2,17),"none","manylinux_2_17_x86_64"), + ("non_monotonic_keep","x86_64",(2,17),"non_monotonic","manylinux_2_17_x86_64"), + ("non_monotonic_drop","x86_64",(2,17),"non_monotonic","manylinux_2_16_x86_64"), + ("legacy1_true","x86_64",(2,17),"manylinux1_compatible_true","manylinux1_x86_64"), + ("legacy1_false","x86_64",(2,17),"manylinux1_compatible_false","manylinux1_x86_64"), + ("legacy2010_true","x86_64",(2,17),"manylinux2010_compatible_true","manylinux2010_x86_64"), + ("legacy2010_false","x86_64",(2,17),"manylinux2010_compatible_false","manylinux2010_x86_64"), + ("legacy2014_true","x86_64",(2,17),"manylinux2014_compatible_true","manylinux2014_x86_64"), + ("legacy2014_false","x86_64",(2,17),"manylinux2014_compatible_false","manylinux2014_x86_64"), + ("raising","x86_64",(2,17),"raising","manylinux_2_17_x86_64")] +result={} +for name,arch,release,mode,tag in cases: + try: result[name]={"member":tag in generated(arch,release,module(mode)),"raised":False} + except RuntimeError: result[name]={"member":False,"raised":True} +print(json.dumps(result,sort_keys=True,separators=(",",":")))` + output, err := exec.Command(python, "-I", "-c", program).CombinedOutput() + if err != nil { + t.Fatalf("selected pip manylinux generator failed: %v: %s", err, output) + } + var results map[string]struct { + Member bool `json:"member"` + Raised bool `json:"raised"` + } + if err := json.Unmarshal(output, &results); err != nil { + t.Fatalf("decode hook results: %v: %s", err, output) + } + want := map[string]bool{ + "absent": true, "true": true, "false": false, "none": true, + "non_monotonic_keep": true, "non_monotonic_drop": false, + "legacy1_true": true, "legacy1_false": false, + "legacy2010_true": true, "legacy2010_false": false, + "legacy2014_true": true, "legacy2014_false": false, + } + for name, member := range want { + if result, ok := results[name]; !ok || result.Raised || result.Member != member { + t.Errorf("%s = %#v, found %v, want member %v", name, result, ok, member) + } + } + if result, ok := results["raising"]; !ok || !result.Raised { + t.Fatalf("raising hook = %#v, found %v", result, ok) + } +} diff --git a/internal/providers/python/materialization.go b/internal/providers/python/materialization.go index c07df163..439dac41 100644 --- a/internal/providers/python/materialization.go +++ b/internal/providers/python/materialization.go @@ -10,7 +10,7 @@ import ( ) const ( - pythonMaterializationEnvironment = "python-v1" + pythonMaterializationEnvironment = "python-v2" pythonMaterializationUmask = "0022" pythonScriptMountID = "script" pythonWheelMountID = "wheels" diff --git a/internal/providers/python/materialization_script.go b/internal/providers/python/materialization_script.go index 7055f984..5fe6fb48 100644 --- a/internal/providers/python/materialization_script.go +++ b/internal/providers/python/materialization_script.go @@ -12,7 +12,7 @@ import ( "github.com/omry/reploy/internal/providerstore" ) -const materializationScriptLogicalPath = "scripts/python-materialize-v1.sh" +const materializationScriptLogicalPath = "scripts/python-materialize-v2.sh" const materializationScriptV1 = `#!/bin/sh set -eu diff --git a/internal/providers/python/materialization_test.go b/internal/providers/python/materialization_test.go index 6012e214..eb193bde 100644 --- a/internal/providers/python/materialization_test.go +++ b/internal/providers/python/materialization_test.go @@ -61,7 +61,7 @@ func TestComponentProviderMaterializeBuildsClosedOfflineTransaction(t *testing.T wantArgv := []providerapi.TypedArgument{ {Kind: providerapi.TypedArgumentValidatedExecutable, ExecutableID: "carrier"}, {Kind: providerapi.TypedArgumentLiteral, Literal: "-eu"}, - {Kind: providerapi.TypedArgumentMountedArtifact, MountID: "script", RelativePath: "scripts/python-materialize-v1.sh"}, + {Kind: providerapi.TypedArgumentMountedArtifact, MountID: "script", RelativePath: "scripts/python-materialize-v2.sh"}, {Kind: providerapi.TypedArgumentValidatedExecutable, ExecutableID: "interpreter"}, {Kind: providerapi.TypedArgumentGeneratedExecutable, GeneratedID: "venv_python"}, {Kind: providerapi.TypedArgumentLiteral, Literal: "/opt/reploy/providers/python/application"}, @@ -98,7 +98,7 @@ func TestComponentProviderMaterializeRejectsBundleInterpreterDrift(t *testing.T) if err != nil { t.Fatal(err) } - bundle.Interpreter.Facts = CanonicalInterpreterFactsV1("3.12.9") + bundle.Interpreter.Facts = CanonicalInterpreterFactsV2(testInterpreterFactsV2("3.12.9", nil, nil)) data, err := CanonicalBundleDataV1(request.Component, bundle) if err != nil { t.Fatal(err) diff --git a/internal/providers/python/owner.go b/internal/providers/python/owner.go index 4d818234..bf0b03a5 100644 --- a/internal/providers/python/owner.go +++ b/internal/providers/python/owner.go @@ -14,29 +14,61 @@ import ( ) const ( - ProfileFactsSchemaV1 = "python-profile-facts-v1" - InterpreterFactsSchemaV1 = "python-interpreter-facts-v1" + ProfileFactsSchemaV2 = "python-profile-facts-v2" ConsoleScriptOutputSchemaV1 = "python-console-script-v1" ) -func CanonicalProfileFactsV1(component string, sources []providers.ResolvedSourceInput) providers.CanonicalProviderData { +func CanonicalProfileFactsV2(component string, sources []providers.ResolvedSourceInput) providers.CanonicalProviderData { values := make([]any, 0, len(sources)) for _, source := range sources { values = append(values, sourceValue(source)) } return providers.CanonicalProviderData{ - Schema: ProfileFactsSchemaV1, + Schema: ProfileFactsSchemaV2, Value: canonical.Object{"component": component, "sources": values}, } } -func CanonicalInterpreterFactsV1(version string) providers.CanonicalProviderData { +func CanonicalInterpreterFactsV2(facts InterpreterInspectionFactsV2) providers.CanonicalProviderData { return providers.CanonicalProviderData{ - Schema: InterpreterFactsSchemaV1, - Value: canonical.Object{"consumer_kind": "python", "version": version}, + Schema: InterpreterFactsSchemaV2, + Value: canonical.Object{ + "version": facts.Version, "implementation": facts.Implementation, "abi": facts.ABI, + "libc": facts.Libc, "libc_major": facts.LibcMajor, "libc_minor": facts.LibcMinor, + "tested_tags": append([]string{}, facts.TestedTags...), + "compatible_tags": append([]string{}, facts.CompatibleTags...), + }, } } +// DecodeInterpreterFactsV2 validates and decodes the complete interpreter +// evidence carried by Python profiles and bundles. +func DecodeInterpreterFactsV2(data providers.CanonicalProviderData) (InterpreterInspectionFactsV2, error) { + if data.Schema != InterpreterFactsSchemaV2 || len(data.Value) != 8 { + return InterpreterInspectionFactsV2{}, fmt.Errorf("Python interpreter facts must use schema %q and the exact value shape", InterpreterFactsSchemaV2) + } + encoded, err := canonical.Marshal(data.Value) + if err != nil { + return InterpreterInspectionFactsV2{}, err + } + decoder := json.NewDecoder(bytes.NewReader(encoded)) + decoder.DisallowUnknownFields() + var facts InterpreterInspectionFactsV2 + if err := decoder.Decode(&facts); err != nil { + return InterpreterInspectionFactsV2{}, fmt.Errorf("decode Python interpreter facts: %w", err) + } + if err := requireJSONEOF(decoder); err != nil { + return InterpreterInspectionFactsV2{}, err + } + if err := validateInspectionFactsV2(facts, facts.TestedTags); err != nil { + return InterpreterInspectionFactsV2{}, err + } + if err := requireCanonicalData(data, CanonicalInterpreterFactsV2(facts), "Python interpreter facts"); err != nil { + return InterpreterInspectionFactsV2{}, err + } + return facts, nil +} + func ValidateRequirementProfileV1(profile providers.RequirementProfile) error { if profile.Provider != blueprint.ComponentTypePython { return fmt.Errorf("Python profile provider must be %q", blueprint.ComponentTypePython) @@ -47,7 +79,7 @@ func ValidateRequirementProfileV1(profile providers.RequirementProfile) error { if err != nil { return fmt.Errorf("Python profile request: %w", err) } - component, _, err := decodeProfileFactsV1(profile.Facts) + component, _, err := decodeProfileFactsV2(profile.Facts) if err != nil { return err } @@ -72,36 +104,32 @@ func ValidateRequirementProfileV1(profile providers.RequirementProfile) error { if interpreter.RequirementID != "interpreter" { return fmt.Errorf("Python profile selected executable must be the interpreter") } - if interpreter.Facts.Schema != InterpreterFactsSchemaV1 || len(interpreter.Facts.Value) != 2 { - return fmt.Errorf("Python interpreter facts must use schema %q and the exact value shape", InterpreterFactsSchemaV1) - } - kind, kindOK := interpreter.Facts.Value["consumer_kind"].(string) - version, versionOK := interpreter.Facts.Value["version"].(string) - if !kindOK || kind != "python" || !versionOK { - return fmt.Errorf("Python interpreter facts have invalid fields") + facts, err := DecodeInterpreterFactsV2(interpreter.Facts) + if err != nil { + return err } - if _, valid := parseReleaseVersion(version); !valid { - return fmt.Errorf("Python interpreter version %q is not a normalized release version", version) + if _, valid := parseReleaseVersion(facts.Version); !valid { + return fmt.Errorf("Python interpreter version %q is not a normalized release version", facts.Version) } - return requireCanonicalData(interpreter.Facts, CanonicalInterpreterFactsV1(version), "Python interpreter facts") + return nil } -// RequirementProfileSelectedSourcesV1 returns the selected local sources +// RequirementProfileSelectedSourcesV2 returns the selected local sources // already bound into a validated Python profile. -func RequirementProfileSelectedSourcesV1(profile providers.RequirementProfile) ([]providers.ResolvedSourceInput, error) { +func RequirementProfileSelectedSourcesV2(profile providers.RequirementProfile) ([]providers.ResolvedSourceInput, error) { if err := ValidateRequirementProfileV1(profile); err != nil { return nil, err } - _, sources, err := decodeProfileFactsV1(profile.Facts) + _, sources, err := decodeProfileFactsV2(profile.Facts) if err != nil { return nil, err } return append([]providers.ResolvedSourceInput{}, sources...), nil } -func decodeProfileFactsV1(data providers.CanonicalProviderData) (string, []providers.ResolvedSourceInput, error) { - if data.Schema != ProfileFactsSchemaV1 || len(data.Value) != 2 { - return "", nil, fmt.Errorf("Python profile facts must use schema %q and the exact value shape", ProfileFactsSchemaV1) +func decodeProfileFactsV2(data providers.CanonicalProviderData) (string, []providers.ResolvedSourceInput, error) { + if data.Schema != ProfileFactsSchemaV2 || len(data.Value) != 2 { + return "", nil, fmt.Errorf("Python profile facts must use schema %q and the exact value shape", ProfileFactsSchemaV2) } var wire struct { Component string `json:"component"` @@ -136,7 +164,7 @@ func decodeProfileFactsV1(data providers.CanonicalProviderData) (string, []provi return "", nil, err } } - expected := CanonicalProfileFactsV1(wire.Component, wire.Sources) + expected := CanonicalProfileFactsV2(wire.Component, wire.Sources) if err := requireCanonicalData(data, expected, "Python profile facts"); err != nil { return "", nil, err } diff --git a/internal/providers/python/prepared_bundle.go b/internal/providers/python/prepared_bundle.go index 197399f3..2af6be74 100644 --- a/internal/providers/python/prepared_bundle.go +++ b/internal/providers/python/prepared_bundle.go @@ -161,7 +161,7 @@ func (resolver WheelNodeResolver) Resolve( SelectedExecutables: []providerapi.ExecutableEvidence{interpreter}, SelectedFiles: []providerapi.FileEvidence{}, Platform: input.Platform, - Facts: CanonicalProfileFactsV1(request.Component, selectedSources), + Facts: CanonicalProfileFactsV2(request.Component, selectedSources), } profileDigest, err := providerapi.RequirementProfileDigest(profile, ValidateRequirementProfileV1) if err != nil { diff --git a/internal/providers/python/prepared_bundle_test.go b/internal/providers/python/prepared_bundle_test.go index 5362e1aa..9cf1030c 100644 --- a/internal/providers/python/prepared_bundle_test.go +++ b/internal/providers/python/prepared_bundle_test.go @@ -240,7 +240,7 @@ func TestWheelNodeResolverRequiresResolvedSourceArtifactDigest(t *testing.T) { if len(result.SelectedSources) != 1 || !reflect.DeepEqual(result.SelectedSources[0], source) { t.Fatalf("selected sources = %#v", result.SelectedSources) } - component, profileSources, err := decodeProfileFactsV1(result.Profile.Facts) + component, profileSources, err := decodeProfileFactsV2(result.Profile.Facts) if err != nil { t.Fatal(err) } @@ -304,7 +304,7 @@ func preparedNodeTestPlan(t *testing.T, requirement string) (providerapi.Provide catalogEvidence.Terminal.RequirementID = "" catalogEvidence.Facts = providerapi.CanonicalProviderData{Schema: "base-python-facts-v1", Value: canonical.Object{}} selectedEvidence := schemaTestInterpreterEvidence() - selectedEvidence.Facts = CanonicalInterpreterFactsV1("3.13.2") + selectedEvidence.Facts = CanonicalInterpreterFactsV2(testInterpreterFactsV2("3.13.2", nil, nil)) catalog := []providerapi.RealizedOutput{{ SupplierComponent: "base", SupplierNode: "base", Name: "python", Candidate: providerapi.ExecutableCandidate{InvocationPath: "/usr/bin/python3", Provenance: baseOutput.Provenance}, diff --git a/internal/providers/python/resolver_recipe.go b/internal/providers/python/resolver_recipe.go index 1bb89b2d..643e5961 100644 --- a/internal/providers/python/resolver_recipe.go +++ b/internal/providers/python/resolver_recipe.go @@ -26,8 +26,9 @@ func WheelResolverArgv( sources []providers.ResolvedSourceInput, reusable []providerstore.ArtifactDescriptor, ) ([]string, error) { - if interpreter == "" || !path.IsAbs(interpreter) || path.Clean(interpreter) != interpreter || strings.Contains(interpreter, `\`) { - return nil, fmt.Errorf("Python wheel resolver interpreter must be a normalized absolute path") + prefix, err := IsolatedInterpreterCommandPrefixV2(interpreter) + if err != nil { + return nil, fmt.Errorf("Python wheel resolver interpreter: %w", err) } decoded, err := decodeCanonicalProviderRequestV1(request) if err != nil { @@ -37,11 +38,10 @@ func WheelResolverArgv( if err != nil { return nil, err } - argv := []string{ - interpreter, "-m", "pip", "--disable-pip-version-check", + argv := append(prefix, + "-m", "pip", "--disable-pip-version-check", "wheel", "--no-cache-dir", "--progress-bar", "off", - "--find-links", ResolverInputDirectory, "--wheel-dir", ResolverOutputDirectory, - } + "--find-links", ResolverInputDirectory, "--wheel-dir", ResolverOutputDirectory) if len(constraints) != 0 { argv = append(argv, "--constraint", ResolverSourceConstraintsPath) } diff --git a/internal/providers/python/resolver_recipe_test.go b/internal/providers/python/resolver_recipe_test.go index cdcaff5e..b3590d94 100644 --- a/internal/providers/python/resolver_recipe_test.go +++ b/internal/providers/python/resolver_recipe_test.go @@ -34,7 +34,7 @@ func TestWheelResolverArgvUsesOnePipClosureWithOptionalSourceConstraints(t *test t.Fatal(err) } want := []string{ - "/usr/bin/python3", "-m", "pip", "--disable-pip-version-check", "wheel", "--no-cache-dir", + "/usr/bin/python3", "-I", "-m", "pip", "--disable-pip-version-check", "wheel", "--no-cache-dir", "--progress-bar", "off", "--find-links", ResolverInputDirectory, "--wheel-dir", ResolverOutputDirectory, "--constraint", ResolverSourceConstraintsPath, "demo>=1", } diff --git a/internal/providers/python/schema_test.go b/internal/providers/python/schema_test.go index 8e825ad6..5c4eaf6a 100644 --- a/internal/providers/python/schema_test.go +++ b/internal/providers/python/schema_test.go @@ -118,7 +118,7 @@ func TestPythonOwnerValidatorsBindProfileAndBundlePayload(t *testing.T) { t.Fatal(err) } interpreter := schemaTestInterpreterEvidence() - interpreter.Facts = CanonicalInterpreterFactsV1("3.13.2") + interpreter.Facts = CanonicalInterpreterFactsV2(testInterpreterFactsV2("3.13.2", nil, nil)) platform, err := blueprint.ParsePlatform("linux/amd64") if err != nil { t.Fatal(err) @@ -133,7 +133,24 @@ func TestPythonOwnerValidatorsBindProfileAndBundlePayload(t *testing.T) { ProviderData: providers.CanonicalProviderData{Schema: request.Schema, Value: request.Value}, }, SelectedExecutables: []providers.ExecutableEvidence{interpreter}, SelectedFiles: []providers.FileEvidence{}, - Platform: platform, Facts: CanonicalProfileFactsV1("application", []providers.ResolvedSourceInput{}), + Platform: platform, Facts: CanonicalProfileFactsV2("application", []providers.ResolvedSourceInput{}), + } + legacyProfile := profile + legacyProfile.Facts = providers.CanonicalProviderData{ + Schema: "python-profile-facts-v1", + Value: canonical.Object{"component": "application", "sources": []any{}}, + } + if err := ValidateRequirementProfileV1(legacyProfile); err == nil || !strings.Contains(err.Error(), ProfileFactsSchemaV2) { + t.Fatalf("legacy profile facts error = %v", err) + } + legacyInterpreter := profile + legacyInterpreter.SelectedExecutables = append([]providers.ExecutableEvidence{}, profile.SelectedExecutables...) + legacyInterpreter.SelectedExecutables[0].Facts = providers.CanonicalProviderData{ + Schema: "python-interpreter-facts-v1", + Value: canonical.Object{"consumer_kind": "python", "version": "3.13.2"}, + } + if err := ValidateRequirementProfileV1(legacyInterpreter); err == nil || !strings.Contains(err.Error(), InterpreterFactsSchemaV2) { + t.Fatalf("legacy interpreter facts error = %v", err) } profileDigest, err := providers.RequirementProfileDigest(profile, ValidateRequirementProfileV1) if err != nil { @@ -184,7 +201,7 @@ func TestPythonOwnerValidatorsBindProfileAndBundlePayload(t *testing.T) { t.Fatal(err) } wrongRecipe := payload - wrongRecipe.RecipeVersion = "python-v2" + wrongRecipe.RecipeVersion = "python-v1" if err := ValidateResolvedBundlePayloadV1(wrongRecipe); err == nil || !strings.Contains(err.Error(), "recipe version") { t.Fatalf("recipe mismatch error = %v", err) } @@ -238,6 +255,6 @@ func schemaTestInterpreterEvidence() providers.ExecutableEvidence { Schema: providers.PortableAccessSchemaV1, Profile: providers.PortableOutputAccessV1, Paths: []providers.AccessPathEvidence{{Path: "/usr/bin/python3", Kind: "regular", Mode: "0755", Required: "other-read-execute"}}, }, - Facts: canonical.Envelope{Schema: "python-interpreter-facts-v1", Value: canonical.Object{}}, + Facts: CanonicalInterpreterFactsV2(testInterpreterFactsV2("3.12.9", nil, nil)), } } diff --git a/internal/providers/registry/validation.go b/internal/providers/registry/validation.go index f684506a..dde38fa8 100644 --- a/internal/providers/registry/validation.go +++ b/internal/providers/registry/validation.go @@ -64,7 +64,7 @@ func RequirementProfileSelectedSourcesV1( } return []providers.ResolvedSourceInput{}, nil case blueprint.ComponentTypePython: - return pythonprovider.RequirementProfileSelectedSourcesV1(profile) + return pythonprovider.RequirementProfileSelectedSourcesV2(profile) default: return nil, fmt.Errorf("provider %q does not have a requirement profile", provider) }