diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index c3f1b090b..e6c895ede 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -20,6 +20,7 @@ import shutil import click from .. import _yaml +from .._frontend.app import App from .._exceptions import BstError, LoadError, AppError, RemoteError from .complete import main_bashcomplete, complete_path, CompleteUnhandled from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope @@ -377,8 +378,6 @@ def cli(context, **kwargs): user preferences configuration file. """ - from .app import App - # Create the App, giving it the main arguments context.obj = App.create(dict(kwargs)) context.call_on_close(context.obj.cleanup) @@ -686,6 +685,13 @@ def show(app, elements, deps, except_, order, format_): metavar="HOSTPATH PATH", help="Mount a file or directory into the sandbox", ) +@click.option( + "--with", + "other_targets", + type=click.Path(readable=False), + multiple=True, + help="A additional target to stage into an element's sandbox environment", +) @click.option("--isolate", is_flag=True, help="Create an isolated build sandbox") @click.option( "--use-buildtree", @@ -723,8 +729,9 @@ def show(app, elements, deps, except_, order, format_): @click.argument("command", type=click.STRING, nargs=-1) @click.pass_obj def shell( - app, + app: App, target, + other_targets, command, mount, isolate, @@ -748,13 +755,40 @@ def shell( otherwise bst may respond to them instead. e.g. \b - bst shell example.bst -- df -h + bst shell base.bst -- df -h Use the --build option to create a temporary sysroot for building the element instead. + Use the --with option to stage the artifacts of other elements + into the temporary sysroot to make them available to run e.g. + + \b + bst shell --with base.bst example.bst -- cat example.txt + If no COMMAND is specified, the default is to attempt to run an interactive shell. + + # Examples: + + \b + # Attempt to run an interactive shell with example.bst + bst shell example.bst + # Attempt to run an df -h with example.bst + bst shell example.bst -- df h + # In a workspace directory, attempt to shell into the workspace element + bst shell + # Attempt to run cat from base.bst to read example.txt from example.bst + bst shell --with base.bst example.bst -- cat example.txt + # Attempt to run an interactive shell with the sources and all dependencies of example.bst + bst shell --build example.bst + + For all examples on this page: + + - example.bst is a simple import element with no dependencies + that imports a file called example.txt + - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc). + """ # Buildtree can only be used with build shells @@ -764,6 +798,7 @@ def shell( scope = _Scope.BUILD if build_ else _Scope.RUN with app.initialized(): + assert app.stream, "Must have Stream initialised" if not target: target = app.stream.get_default_target() if not target: @@ -772,19 +807,35 @@ def shell( mounts = [_HostMount(path, host_path) for host_path, path in mount] try: - exitcode = app.stream.shell( - target, - scope, - app.shell_prompt, - mounts=mounts, - isolate=isolate, - command=command, - usebuildtree=cli_buildtree, - artifact_remotes=artifact_remotes, - source_remotes=source_remotes, - ignore_project_artifact_remotes=ignore_project_artifact_remotes, - ignore_project_source_remotes=ignore_project_source_remotes, - ) + if other_targets: + exitcode = app.stream.shell_with( + target, + other_targets, + scope, + app.shell_prompt, + mounts=mounts, + isolate=isolate, + command=command, + usebuildtree=cli_buildtree, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) + else: + exitcode = app.stream.shell( + target, + scope, + app.shell_prompt, + mounts=mounts, + isolate=isolate, + command=command, + usebuildtree=cli_buildtree, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) except BstError as e: raise AppError("Error launching shell: {}".format(e), detail=e.detail, reason=e.reason) from e diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index af98504cc..df454ec98 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -14,14 +14,19 @@ # Authors: # Tristan Van Berkom +from buildstream import utils +import sys +import tempfile +from typing import Callable, Generator import os -from contextlib import suppress +from contextlib import suppress, contextmanager +from ruamel.yaml import CommentedMap from .._exceptions import LoadError from ..exceptions import LoadErrorReason from .. import _yaml from ..element import Element -from ..node import Node +from ..node import Node, MappingNode from .._profile import Topics, PROFILER from .._includes import Includes from .._utils import valid_chars_name @@ -73,7 +78,7 @@ def __init__(self, project, *, parent=None, provenance_node=None): self._links = {} # Dict of link target target paths indexed by link element paths self._loaders = {} # Dict of junction loaders self._loader_search_provenances = {} # Dictionary of provenance nodes of ongoing child loader searches - + self._fullpath_overrides: dict[str, str] = {} # Dictionary: Original Path, Replacement Path self._includes = Includes(self, copy_tree=True) assert project.name is not None @@ -251,10 +256,58 @@ def foreach_parent(parent): for parent in self._alternative_parents: yield from foreach_parent(parent) + # temporary_modified_element() + # + # Temporarily modify an element by loading the element and applying modify_elements_function to make the modifications + # + # + # Args: + # target (str): The element-path relative bst file + # modify_element_function (Callable[[CommentedMap],None]): A function to modify a given CommentedMap + # + @contextmanager + def temporary_modified_element( + self, target: str, modify_element_function: Callable[[CommentedMap], None] + ) -> Generator[None, None, None]: + + _, target_name, target_loader = self._parse_name(target, MappingNode.from_dict({})) + + target_path = os.path.join(target_loader._basedir, target_name) + target_node: CommentedMap = _yaml.roundtrip_load(target_path) + + modify_element_function(target_node) + + with utils._tempnamedfile(prefix=f"{target_name.replace('/','_')}_temp", suffix=".bst" ) as temp_target_file: + _yaml.roundtrip_dump(target_node, temp_target_file) + temp_target_file.flush() + target_loader._set_fullpath_override(target_name, temp_target_file.name) + + try: + yield + finally: + target_loader._set_fullpath_override(target_name, None) + ########################################### # Private Methods # ########################################### + # _set_fullpath_override() + # + # Set an fullpath override for a element-path relative bst file + # + # This enables runtime modified elements to be pulled from a temporary directory + # Passing None as a fullpath remove the entry + # + # Args: + # filename (str): The element-path relative bst file + # fullpath (str|None): A fullpath to the bst file, or None + # + def _set_fullpath_override(self, filename: str, fullpath: str | None): + if fullpath: + self._fullpath_overrides[filename] = fullpath + else: + self._fullpath_overrides.pop(filename, None) + # _load_file_no_deps(): # # Load a bst file as a LoadElement @@ -275,8 +328,8 @@ def _load_file_no_deps(self, filename, provenance_node=None, only_first_pass=Fal self._assert_element_name(filename, provenance_node) - # Load the data and process any conditional statements therein - fullpath = os.path.join(self._basedir, filename) + fullpath = self._fullpath_overrides.get(filename, os.path.join(self._basedir, filename)) + try: node = _yaml.load( fullpath, shortname=filename, copy_tree=self.load_context.rewritable, project=self.project @@ -1014,7 +1067,9 @@ def _shallow_load_path(self, path, provenance_node): # - (str): name of the element # - (Loader): loader for sub-project # - def _parse_name(self, name, provenance_node, *, load_subprojects=True): + def _parse_name( + self, name: str, provenance_node: MappingNode, *, load_subprojects: bool = True + ) -> tuple[str | None, str, "Loader"]: # We allow to split only once since deep junctions names are forbidden. # Users who want to refer to elements in sub-sub-projects are required # to create junctions on the top level project. diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index a1d88d667..7bc34c2f4 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -16,6 +16,7 @@ # Jürg Billeter # Tristan Maat + import itertools import os import sys @@ -27,7 +28,10 @@ from contextlib import contextmanager, suppress from collections import deque from typing import List, Tuple, Optional, Iterable, Callable +from ruamel.yaml import CommentedMap + +from ._context import Context from ._artifactelement import verify_artifact_ref, ArtifactElement from ._artifactproject import ArtifactProject from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError @@ -44,7 +48,7 @@ ) from .element import Element from ._profile import Topics, PROFILER -from ._project import ProjectRefStorage +from ._project import ProjectRefStorage, Project from ._remotespec import RemoteSpec from ._state import State from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount @@ -79,11 +83,11 @@ def __init__( # # Private members # - self._context = context + self._context: Context = context self._artifacts = None self._elementsourcescache = None self._sourcecache = None - self._project = None + self._project: Optional[Project] = None self._state = State(session_start) # Owned by Stream, used by Core to set state self._notification_queue = deque() @@ -163,7 +167,7 @@ def load_selection( ignore_project_artifact_remotes: bool = False, ignore_project_source_remotes: bool = False, need_state: bool = True, - ): + ) -> list[Element]: with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)): target_objects = self._load( targets, @@ -235,6 +239,53 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source task.add_current_progress() + # shell_with() + # + # Run a shell with other targets. + # + # Automatically creates a temporary target based on 'target' with 'other_targets' as runtime or build dependencies. + # + # Note: Method will build the temporary target, before entering into it's shell. + # + # Args: + # target (str): The name of the element to run the shell for + # other_targets: (Iterable[str]): The name of the other elements to run the shell with. + # scope: _Scope: Either BUILD or RUN + # *args, **kwargs: Passed to shell() untouched. + # + # Returns: + # (int): The exit code of the launched shell + # + def shell_with(self, target: str, other_targets: Iterable[str], scope: _Scope, *args, **kwargs): + + assert self._project, "Must have a project" + assert self._project.loader, "Project must have loader" + + def add_deps_to_element(target_node: CommentedMap): + if scope == _Scope.RUN: + r_depends = target_node.get("runtime-depends", []) + + for other_target in other_targets: + r_depends.append(other_target) + + target_node["runtime-depends"] = r_depends + elif scope == _Scope.BUILD: + r_depends = target_node.get("build-depends", []) + + for other_target in other_targets: + r_depends.append(other_target) + + target_node["build-depends"] = r_depends + else: + raise StreamError( + "Only BUILD and RUN scopes are supported", + detail="Use the --build and --use-buildtree options to shell into a build tree", + reason="only-build-run-supported", + ) + + with self._project.loader.temporary_modified_element(target, add_deps_to_element): + return self.shell(target, scope, *args, **kwargs) + # shell() # # Run a shell @@ -243,6 +294,7 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source # target: The name of the element to run the shell for # scope: The scope for the shell, only BUILD or RUN are valid (_Scope) # prompt: A function to return the prompt to display in the shell + # other_targets (Iterable[str]): The name of other elements to stage in the shell # unique_id: (str): A unique_id to use to lookup an Element instance # mounts: Additional directories to mount into the sandbox # isolate (bool): Whether to isolate the environment like we do in builds @@ -1043,6 +1095,7 @@ def workspace_open( self.workspace_close(target._get_full_name(), remove_dir=not no_checkout) if not custom_dir: + assert self._context.workspacedir, "Must have workspace dir" directory = os.path.abspath(os.path.join(self._context.workspacedir, target.name)) if directory[-4:] == ".bst": directory = directory[:-4] @@ -2117,6 +2170,8 @@ def _expand_and_classify_targets( # project directory and element path prefix, to produce only element names. # all_elements = [] + assert self._project, "Must have a project" + assert self._project.element_path, "Must have a project" element_path_length = len(self._project.element_path) + 1 for dirpath, _, filenames in os.walk(self._project.element_path): for filename in filenames: @@ -2136,6 +2191,7 @@ def _expand_and_classify_targets( # Glob the artifact names and add the results to the set # + assert self._artifacts, "Must have artifacts" for glob in artifact_globs: glob_results = self._artifacts.list_artifacts(glob=glob) for artifact_name in glob_results: diff --git a/src/buildstream/_yaml.pyi b/src/buildstream/_yaml.pyi index 224abc51a..b301cab68 100644 --- a/src/buildstream/_yaml.pyi +++ b/src/buildstream/_yaml.pyi @@ -12,7 +12,10 @@ # limitations under the License. # from typing import Optional +from ruamel.yaml import CommentedMap from .node import MappingNode def load(filename: str, shortname: str, copy_tree: bool = False, project: Optional[object] = None) -> MappingNode: ... +def roundtrip_load(filename: str, *, allow_missing: bool = False) -> CommentedMap: ... +def roundtrip_dump(contents, file): ... diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 621568e63..a0b38ef94 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -62,6 +62,9 @@ --------------- """ +# For 3.7+ support, not necessary and deprecated in 3.14+ +from __future__ import annotations + import os import re import stat @@ -74,6 +77,7 @@ from threading import Lock from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence + from pyroaring import BitMap # pylint: disable=no-name-in-module from . import _yaml @@ -2058,6 +2062,7 @@ def _push(self): # prompt (str): A suitable prompt string for PS1 # command (list): An argv to launch in the sandbox # usebuildtree (bool): Use the buildtree as its source + # other_elements (List[Element]): Optional list of other runtime elements to stage in the sandbox # # Returns: Exit code def _shell( diff --git a/src/buildstream/types.pyi b/src/buildstream/types.pyi index 8a75b9f13..db9d59d06 100644 --- a/src/buildstream/types.pyi +++ b/src/buildstream/types.pyi @@ -116,10 +116,10 @@ class OverlapAction(Enum): IGNORE: str class _Scope(Enum): - ALL: int - BUILD: int - RUN: int - NONE: int + ALL = 1 + BUILD = 2 + RUN = 3 + NONE = 4 class _KeyStrength(Enum): STRONG: int diff --git a/tests/integration/shell.py b/tests/integration/shell.py index 63c7af8b3..b8d24c424 100644 --- a/tests/integration/shell.py +++ b/tests/integration/shell.py @@ -16,12 +16,15 @@ # pylint: disable=redefined-outer-name import os +from typing import Dict, List, Tuple import uuid + import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import +from buildstream._testing.runcli import CliIntegration from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN from buildstream.exceptions import ErrorDomain from buildstream import utils @@ -46,11 +49,26 @@ # mount (tuple): A (host, target) tuple for the `--mount` option # element (str): The element to build and run a shell with # isolate (bool): Whether to pass --isolate to `bst shell` +# other_elements (list(str)): Other elements to stage in the sandbox # -def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False): +def execute_shell( + cli: CliIntegration, + project: str, + command: List[str], + *, + config: None | Dict = None, + mount: Tuple[str, str] | None = None, + element: str = "base.bst", + isolate: bool = False, + other_elements: List[str] | None = None, +): # Ensure the element is built result = cli.run_project_config(project=project, project_config=config, args=["build", element]) assert result.exit_code == 0 + if other_elements is not None: + for other_element in other_elements: + result = cli.run_project_config(project=project, project_config=config, args=["build", other_element]) + assert result.exit_code == 0 args = ["shell"] if isolate: @@ -58,6 +76,9 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element="ba if mount is not None: host_path, target_path = mount args += ["--mount", host_path, target_path] + if other_elements is not None: + for other_element in other_elements: + args += ["--with", other_element] args += [element, "--", *command] return cli.run_project_config(project=project, project_config=config, args=args) @@ -86,6 +107,28 @@ def test_executable(cli, datafiles): assert result.output == "Horseys!\n" +# Test staging and running additional targets in the shell of the main target for debugging. +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_with_other_targets(cli, datafiles): + project = str(datafiles) + + # Show we can't cat in a shell for manual/import-file.bst + result = execute_shell(cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst") + assert ( + result.exit_code == -1 + ), "Shouldn't be able to read content of test.txt as manual/import-file.bst is a simple import element with no dependencies" + + # Show we can now cat with base.bst in a shell for manual/import-file.bst + result = execute_shell( + cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst", other_elements=["base.bst"] + ) + assert ( + result.exit_code == 0 + ), "Should be able to read content of test.txt as we now stage in base.bst that provides /bin/cat" + assert result.output == "This is a test\n" + + # Test shell environment variable explicit assignments @pytest.mark.parametrize("animal", [("Horse"), ("Pony")]) @pytest.mark.datafiles(DATA_DIR)