diff --git a/.github/common.env b/.github/common.env index c9dd2d6eb..fd2da1288 100644 --- a/.github/common.env +++ b/.github/common.env @@ -1,6 +1,6 @@ # Shared common variables -CI_IMAGE_VERSION=master-2783218587 +CI_IMAGE_VERSION=master-2856852845 CI_TOXENV_MAIN=py310,py311,py312,py313,py314,py315 CI_TOXENV_PLUGINS=py310-plugins,py311-plugins,py312-plugins,py313-plugins,py314-plugins,py315-plugins CI_TOXENV_ALL="${CI_TOXENV_MAIN},${CI_TOXENV_PLUGINS}" diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst index 3010eb0b7..1b5b178e2 100644 --- a/doc/source/using_config.rst +++ b/doc/source/using_config.rst @@ -853,6 +853,7 @@ Cache server configuration is declared in the following way: override-project-caches: false servers: - url: https://cache-server.com/cache:11001 + protocol: grpc instance-name: main type: all push: true @@ -888,6 +889,13 @@ Attributes Indicates the ``http`` or ``https`` url and optionally the port number of where the cache server is located. + * ``protocol`` + + The protocol to use. This is optional and defaults to ``grpc``, the gRPC-based + `Remote Execution API (REAPI) `_. + For ``storage`` servers where ``push`` is disabled, this can be set to ``http`` + to instead use a `HTTP REST protocol `_. + * ``instance-name`` Instance names separate different shards on the same endpoint (``url``). diff --git a/src/buildstream/_assetcache.py b/src/buildstream/_assetcache.py index 9406b9cf9..0d4a7c476 100644 --- a/src/buildstream/_assetcache.py +++ b/src/buildstream/_assetcache.py @@ -56,6 +56,9 @@ def _configure_protocols(self): # RemoteError: If the upstream has a problem # def _check(self): + if self.spec.protocol != "grpc": + raise RemoteError("Index servers are supported only with the 'grpc' protocol") + request = remote_asset_pb2.FetchBlobRequest() if self.instance_name: request.instance_name = self.instance_name @@ -324,6 +327,12 @@ def setup_remotes(self, specs: Iterable[RemoteSpec], project_specs: Dict[str, Li remote = RemotePair(casd, spec) if remote.error: + if spec.protocol == "http" and "failed to connect to all addresses" in remote.error: + # Received gRPC error message even though protocol was set to 'http' + remote.error = ( + "Your version of buildbox-casd may be too old to support the HTTP REST protocol for CAS" + ) + self.context.messenger.warn("Failed to initialize remote {}: {}".format(spec.url, remote.error)) self._remotes[spec] = remote diff --git a/src/buildstream/_protos/build/buildgrid/local_cas.proto b/src/buildstream/_protos/build/buildgrid/local_cas.proto index 0dc0cbc36..cca5e21cb 100644 --- a/src/buildstream/_protos/build/buildgrid/local_cas.proto +++ b/src/buildstream/_protos/build/buildgrid/local_cas.proto @@ -100,6 +100,21 @@ service LocalContentAddressableStorage { // cleanup is complete. rpc StageTree(stream StageTreeRequest) returns (stream StageTreeResponse) {} + // Export files to a user specified location on the local filesystem. + // + // The way in which files are populated is implementation defined, + // however, the server will guarantee that mutations to these files + // will not corrupt the local cache. + // + // This does not create any directories. The client needs to ensure that + // the parent directory of each output file exists and is writable. + // + // Missing blobs are fetched, if a CAS remote is configured. + // + // Errors: + // * `FAILED_PRECONDITION`: The destination directory does not exist. + rpc ExportFiles(ExportFilesRequest) returns (ExportFilesResponse) {} + // Capture a directory tree from the local filesystem. // // This imports the specified path from the local filesystem into CAS. @@ -138,6 +153,17 @@ service LocalContentAddressableStorage { // workflows where an upload step adds unnecessary latency. rpc CaptureFiles(CaptureFilesRequest) returns (CaptureFilesResponse) {} + // Hash files from the local filesystem. + // + // This asks the local CAS server to hash the file on behalf of the + // client and populate the inode cache for efficent subqeuent hashing + // of the same file. + // + // Unlike CaptureFiles which always store the file content either to + // local storage or remote CAS or both. HashFiles only return the hash + // and populate the inode cache. + rpc HashFiles(HashFilesRequest) returns (HashFilesResponse) {} + // Configure remote CAS endpoint. // // This returns a string that can be used as instance_name to access the @@ -153,15 +179,20 @@ service LocalContentAddressableStorage { // specified endpoints in further requests. rpc GetInstanceNameForRemotes(GetInstanceNameForRemotesRequest) returns (GetInstanceNameForRemotesResponse) {} - // Configure sandboxed clients. - // - // This returns a string that can be used as instance_name to access - // this service from clients running in the specified filesystem/mount - // namespace or chroot environment - rpc GetInstanceNameForNamespace(GetInstanceNameForNamespaceRequest) returns (GetInstanceNameForNamespaceResponse) {} - // Query total space used by the local cache. rpc GetLocalDiskUsage(GetLocalDiskUsageRequest) returns (GetLocalDiskUsageResponse) {} + + + // Create a nested server restricted to a directory tree in the local filesystem. + // + // The nested server is started when the server receives the initial request + // and it is ready to be used on the initial (non-error) response from the + // server. + // + // The nested server will shut down when the server either receives an + // additional request (with all fields unset) or when the stream is closed. + // The server will send an additional response after cleanup is complete. + rpc NestedServer(stream NestedServerRequest) returns (stream NestedServerResponse) {} } // A request message for @@ -306,6 +337,26 @@ message UploadTreeRequest { message UploadTreeResponse { } +message RemoteApisSocketConfig { + // The instance of the execution system clients of the additional server + // socket will operate against. + string instance_name = 1; + + // The path, relative to the staging directory, where a server socket should + // be created for access to the Remote Execution API. + string path = 2; + + // If true, allow clients of the additional server socket to update the + // action cache of the specified instance. + bool action_cache_update_enabled = 3; + + // A map from cloned instance name to base instance name. + // The nested server will clone each base instance and expose it + // under the corresponding cloned name (the key). + // Cannot be used together with instance_name. + map instance_map = 4; +} + // A request message for // [LocalContentAddressableStorage.StageTree][build.buildgrid.v2.LocalContentAddressableStorage.StageTree]. message StageTreeRequest { @@ -344,9 +395,9 @@ message StageTreeRequest { // for access without risking corruption of files in the local cache. Credentials access_credentials = 4; - // The path, relative to the staging directory, where a server socket should - // be created for access to the Remote Execution API. - string remote_apis_socket_path = 7; + // Optional configuration of an additional server socket to be created for + // access to the Remote Execution API. + RemoteApisSocketConfig remote_apis_socket = 8; // The commands to run against the tree before it is cleaned. // Each command must be an absolute path to an executable which can be run as @@ -382,6 +433,39 @@ message StageTreeResponse { string path = 1; } +// A request message for +// [LocalContentAddressableStorage.ExportFiles][build.buildgrid.v2.LocalContentAddressableStorage.ExportFiles]. +message ExportFilesRequest { + // The instance of the execution system to operate against. A server may + // support multiple instances of the execution system (with their own workers, + // storage, caches, etc.). The server MAY require use of this field to select + // between them in an implementation-defined fashion, otherwise it can be + // omitted. + string instance_name = 1; + + // Output paths of individual files will be resolved relative to this path. + string path = 2; + + // The individual files to export. + repeated build.bazel.remote.execution.v2.OutputFile output_files = 3; +} + +// A response message for +// [LocalContentAddressableStorage.ExportFiles][build.buildgrid.v2.LocalContentAddressableStorage.ExportFiles]. +message ExportFilesResponse { + // A response corresponding to a single blob that the client tried to export. + message Response { + // The path to which this response corresponds. + string path = 1; + + // The result of attempting to export the file. + google.rpc.Status status = 2; + } + + // The responses to the requests. + repeated Response responses = 1; +} + // A request message for // [LocalContentAddressableStorage.CaptureTree][build.buildgrid.v2.LocalContentAddressableStorage.CaptureTree]. message CaptureTreeRequest { @@ -432,6 +516,12 @@ message CaptureTreeRequest { // For example, a mask of `0222` will result in contents being captured as read-only. // This is only effective when the unix_mode property is captured. google.protobuf.UInt32Value unix_mode_mask = 9; + + // If true, temporarily grant read permissions to files lacking read access + // during capture, then restore original permissions afterwards. + // The files/directories are expected to be owned by the same user running + // the capture. + bool allow_chmod_to_read = 10; } // A response message for @@ -454,6 +544,66 @@ message CaptureTreeResponse { // [Tree][build.bazel.remote.execution.v2.Directory] proto containing the // directory's contents, if successful. build.bazel.remote.execution.v2.Digest root_directory_digest = 4; + + // The accumulated total size of all blobs in the tree + int64 total_size_bytes = 5; + } + + // The responses to the requests. + repeated Response responses = 1; +} + +// A request message for +// [LocalContentAddressableStorage.HashFiles][build.buildgrid.v2.LocalContentAddressableStorage.HashFiles]. +message HashFilesRequest { + // The instance of the execution system to operate against. A server may + // support multiple instances of the execution system (with their own workers, + // storage, caches, etc.). The server MAY require use of this field to select + // between them in an implementation-defined fashion, otherwise it can be + // omitted. + string instance_name = 1; + + // The optional root path to restrict hash to a subtree. + // If specified, `path` will be resolved inside this root. + // No files outside the root will be hashed + string root = 3; + + // The path(s) in the local filesystem to capture. + repeated string path = 2; + + // The properties of path(s) in the local filesystem to capture. + repeated string node_properties = 4; + + // The mask to apply to the files being captured. + // For example, a mask of `0222` will result in contents being captured as read-only. + // This is only effective when the unix_mode property is captured. + google.protobuf.UInt32Value unix_mode_mask = 5; + + // If true, temporarily grant read permissions to files lacking read access + // during capture, then restore original permissions afterwards. + // The files/directories are expected to be owned by the same user running + // the capture. + bool allow_chmod_to_read = 6; +} + +// A response message for +// [LocalContentAddressableStorage.HashFiles][build.buildgrid.v2.LocalContentAddressableStorage.HashFiles]. +message HashFilesResponse { + // A response corresponding to a single blob that the client tried to upload. + message Response { + // The path to which this response corresponds. + string path = 1; + + // The digest of the hashed file's content, if successful. + build.bazel.remote.execution.v2.Digest digest = 2; + + // The result of attempting to hash the file and populate the inode cache. + google.rpc.Status status = 3; + + // True if the hashed file was executable, false otherwise. + bool is_executable = 4; + + build.bazel.remote.execution.v2.NodeProperties node_properties = 5; } // The responses to the requests. @@ -500,6 +650,12 @@ message CaptureFilesRequest { // For example, a mask of `0222` will result in contents being captured as read-only. // This is only effective when the unix_mode property is captured. google.protobuf.UInt32Value unix_mode_mask = 8; + + // If true, temporarily grant read permissions to files lacking read access + // during capture, then restore original permissions afterwards. + // The files/directories are expected to be owned by the same user running + // the capture. + bool allow_chmod_to_read = 9; } // A response message for @@ -612,6 +768,10 @@ message Remote { // If set, don't upload any blobs, action results or assets to the remote. bool read_only = 12; + + // Protocol to use for communication with the server, acceptable values + // are 'grpc' and 'http' (read-only). It defaults to 'grpc'. + string protocol = 13; } // A request message for @@ -637,27 +797,6 @@ message GetInstanceNameForRemotesResponse { } -// A request message for -// [LocalContentAddressableStorage.GetInstanceNameForRemote][build.buildgrid.v2.LocalContentAddressableStorage.GetInstanceNameForNamespace]. -message GetInstanceNameForNamespaceRequest { - // The instance of the execution system to operate against. A server may - // support multiple instances of the execution system (with their own workers, - // storage, caches, etc.). The server MAY require use of this field to select - // between them in an implementation-defined fashion, otherwise it can be - // omitted. - string instance_name = 1; - - // The root path of the mount namespace to restrict capture and staging. - // All paths in requests to the new instance will be resolved inside this root. - string root = 2; -} - -// A response message for -// [LocalContentAddressableStorage.GetInstanceNameForRemote][build.buildgrid.v2.LocalContentAddressableStorage.GetInstanceNameForNamespace]. -message GetInstanceNameForNamespaceResponse { - string instance_name = 1; -} - // A request message for // [LocalContentAddressableStorage.GetLocalDiskUsage][build.buildgrid.v2.LocalContentAddressableStorage.GetLocalDiskUsage]. message GetLocalDiskUsageRequest { @@ -672,3 +811,38 @@ message GetLocalDiskUsageResponse { // Disk quota for the local cache, in bytes. A value of 0 means no quota is set. int64 quota_bytes = 2; } + + +// A request message for +// [LocalContentAddressableStorage.NestedServer][build.buildgrid.v2.LocalContentAddressableStorage.NestedServer]. +message NestedServerRequest { + // The instance of the execution system to operate against. A server may + // support multiple instances of the execution system (with their own workers, + // storage, caches, etc.). The server MAY require use of this field to select + // between them in an implementation-defined fashion, otherwise it can be + // omitted. + string instance_name = 1; + + // The path in the local filesystem that is used as filesystem root in + // requests sent to the nested server. + string path = 2; + + message Credentials { + int64 uid = 1; + int64 gid = 2; + } + + // The UNIX credentials of the processes that will access the nested server. + // This will be used to ensure that the socket has the right access + // permission. + Credentials access_credentials = 3; + + // Configuration of the additional server socket to be created for access to + // the Remote Execution API. + RemoteApisSocketConfig remote_apis_socket = 4; +} + +// A response message for +// [LocalContentAddressableStorage.NestedServer][build.buildgrid.v2.LocalContentAddressableStorage.NestedServer]. +message NestedServerResponse { +} diff --git a/src/buildstream/_protos/build/buildgrid/local_cas_pb2.py b/src/buildstream/_protos/build/buildgrid/local_cas_pb2.py index 30bebeae3..98cf4fbed 100644 --- a/src/buildstream/_protos/build/buildgrid/local_cas_pb2.py +++ b/src/buildstream/_protos/build/buildgrid/local_cas_pb2.py @@ -28,13 +28,15 @@ from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x62uild/buildgrid/local_cas.proto\x12\x0f\x62uild.buildgrid\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x17google/rpc/status.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1egoogle/protobuf/wrappers.proto\"5\n\x1cGetLocalServerDetailsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"Q\n\x12LocalServerDetails\x12\x10\n\x08hostname\x18\x01 \x01(\t\x12\x13\n\x0bprocess_uid\x18\x02 \x01(\x03\x12\x14\n\x0cstorage_root\x18\x03 \x01(\t\"p\n\x18\x46\x65tchMissingBlobsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12=\n\x0c\x62lob_digests\x18\x02 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\xcc\x01\n\x19\x46\x65tchMissingBlobsResponse\x12\x46\n\tresponses\x18\x01 \x03(\x0b\x32\x33.build.buildgrid.FetchMissingBlobsResponse.Response\x1ag\n\x08Response\x12\x37\n\x06\x64igest\x18\x01 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x02 \x01(\x0b\x32\x12.google.rpc.Status\"q\n\x19UploadMissingBlobsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12=\n\x0c\x62lob_digests\x18\x02 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\xce\x01\n\x1aUploadMissingBlobsResponse\x12G\n\tresponses\x18\x01 \x03(\x0b\x32\x34.build.buildgrid.UploadMissingBlobsResponse.Response\x1ag\n\x08Response\x12\x37\n\x06\x64igest\x18\x01 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x02 \x01(\x0b\x32\x12.google.rpc.Status\"\x81\x01\n\x10\x46\x65tchTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12<\n\x0broot_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x18\n\x10\x66\x65tch_file_blobs\x18\x03 \x01(\x08\"\x13\n\x11\x46\x65tchTreeResponse\"h\n\x11UploadTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12<\n\x0broot_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\x14\n\x12UploadTreeResponse\"\xa5\x03\n\x10StageTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12<\n\x0broot_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x0c\n\x04path\x18\x03 \x01(\t\x12I\n\x12\x61\x63\x63\x65ss_credentials\x18\x04 \x01(\x0b\x32-.build.buildgrid.StageTreeRequest.Credentials\x12\x1f\n\x17remote_apis_socket_path\x18\x07 \x01(\t\x12\x1c\n\x14pre_unstage_commands\x18\x05 \x03(\t\x12\x43\n\x0cstaging_mode\x18\x06 \x01(\x0e\x32-.build.buildgrid.StageTreeRequest.StagingMode\x1a\'\n\x0b\x43redentials\x12\x0b\n\x03uid\x18\x01 \x01(\x03\x12\x0b\n\x03gid\x18\x02 \x01(\x03\"6\n\x0bStagingMode\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x08\n\x04\x46USE\x10\x01\x12\x10\n\x0c\x43OPY_OR_LINK\x10\x02\"!\n\x11StageTreeResponse\x12\x0c\n\x04path\x18\x01 \x01(\t\"\xbc\x02\n\x12\x43\x61ptureTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x06 \x01(\t\x12\x0c\n\x04path\x18\x02 \x03(\t\x12\x1a\n\x12\x62ypass_local_cache\x18\x03 \x01(\x08\x12\x17\n\x0fnode_properties\x18\x04 \x03(\t\x12\x12\n\nmove_files\x18\x05 \x01(\x08\x12_\n\x17output_directory_format\x18\x07 \x01(\x0e\x32>.build.bazel.remote.execution.v2.Command.OutputDirectoryFormat\x12\x13\n\x0bskip_upload\x18\x08 \x01(\x08\x12\x34\n\x0eunix_mode_mask\x18\t \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\"\x9c\x02\n\x13\x43\x61ptureTreeResponse\x12@\n\tresponses\x18\x01 \x03(\x0b\x32-.build.buildgrid.CaptureTreeResponse.Response\x1a\xc2\x01\n\x08Response\x12\x0c\n\x04path\x18\x01 \x01(\t\x12<\n\x0btree_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x03 \x01(\x0b\x32\x12.google.rpc.Status\x12\x46\n\x15root_directory_digest\x18\x04 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\xdc\x01\n\x13\x43\x61ptureFilesRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x06 \x01(\t\x12\x0c\n\x04path\x18\x02 \x03(\t\x12\x1a\n\x12\x62ypass_local_cache\x18\x03 \x01(\x08\x12\x17\n\x0fnode_properties\x18\x04 \x03(\t\x12\x12\n\nmove_files\x18\x05 \x01(\x08\x12\x13\n\x0bskip_upload\x18\x07 \x01(\x08\x12\x34\n\x0eunix_mode_mask\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\"\xb8\x02\n\x14\x43\x61ptureFilesResponse\x12\x41\n\tresponses\x18\x01 \x03(\x0b\x32..build.buildgrid.CaptureFilesResponse.Response\x1a\xdc\x01\n\x08Response\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x03 \x01(\x0b\x32\x12.google.rpc.Status\x12\x15\n\ris_executable\x18\x04 \x01(\x08\x12H\n\x0fnode_properties\x18\x06 \x01(\x0b\x32/.build.bazel.remote.execution.v2.NodePropertiesJ\x04\x08\x05\x10\x06\"\x83\x01\n\x1fGetInstanceNameForRemoteRequest\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x15\n\rinstance_name\x18\x02 \x01(\t\x12\x13\n\x0bserver_cert\x18\x03 \x01(\x0c\x12\x12\n\nclient_key\x18\x04 \x01(\x0c\x12\x13\n\x0b\x63lient_cert\x18\x05 \x01(\x0c\"9\n GetInstanceNameForRemoteResponse\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"\x85\x03\n\x06Remote\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x15\n\rinstance_name\x18\x02 \x01(\t\x12\x13\n\x0bserver_cert\x18\x03 \x01(\x0c\x12\x12\n\nclient_key\x18\x04 \x01(\x0c\x12\x13\n\x0b\x63lient_cert\x18\x05 \x01(\x0c\x12\x19\n\x11\x61\x63\x63\x65ss_token_path\x18\x07 \x01(\t\x12?\n\x1c\x61\x63\x63\x65ss_token_reload_interval\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x31\n\x0ekeepalive_time\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bretry_limit\x18\t \x01(\x03\x12.\n\x0bretry_delay\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x32\n\x0frequest_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x11\n\tread_only\x18\x0c \x01(\x08\"\x81\x02\n GetInstanceNameForRemotesRequest\x12\x15\n\rinstance_name\x18\x03 \x01(\t\x12<\n\x1b\x63ontent_addressable_storage\x18\x01 \x01(\x0b\x32\x17.build.buildgrid.Remote\x12-\n\x0cremote_asset\x18\x02 \x01(\x0b\x32\x17.build.buildgrid.Remote\x12-\n\x0c\x61\x63tion_cache\x18\x04 \x01(\x0b\x32\x17.build.buildgrid.Remote\x12*\n\texecution\x18\x05 \x01(\x0b\x32\x17.build.buildgrid.Remote\":\n!GetInstanceNameForRemotesResponse\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"I\n\"GetInstanceNameForNamespaceRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x02 \x01(\t\"<\n#GetInstanceNameForNamespaceResponse\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"\x1a\n\x18GetLocalDiskUsageRequest\"D\n\x19GetLocalDiskUsageResponse\x12\x12\n\nsize_bytes\x18\x01 \x01(\x03\x12\x13\n\x0bquota_bytes\x18\x02 \x01(\x03\x32\xb8\n\n\x1eLocalContentAddressableStorage\x12m\n\x15GetLocalServerDetails\x12-.build.buildgrid.GetLocalServerDetailsRequest\x1a#.build.buildgrid.LocalServerDetails\"\x00\x12l\n\x11\x46\x65tchMissingBlobs\x12).build.buildgrid.FetchMissingBlobsRequest\x1a*.build.buildgrid.FetchMissingBlobsResponse\"\x00\x12o\n\x12UploadMissingBlobs\x12*.build.buildgrid.UploadMissingBlobsRequest\x1a+.build.buildgrid.UploadMissingBlobsResponse\"\x00\x12T\n\tFetchTree\x12!.build.buildgrid.FetchTreeRequest\x1a\".build.buildgrid.FetchTreeResponse\"\x00\x12W\n\nUploadTree\x12\".build.buildgrid.UploadTreeRequest\x1a#.build.buildgrid.UploadTreeResponse\"\x00\x12X\n\tStageTree\x12!.build.buildgrid.StageTreeRequest\x1a\".build.buildgrid.StageTreeResponse\"\x00(\x01\x30\x01\x12Z\n\x0b\x43\x61ptureTree\x12#.build.buildgrid.CaptureTreeRequest\x1a$.build.buildgrid.CaptureTreeResponse\"\x00\x12]\n\x0c\x43\x61ptureFiles\x12$.build.buildgrid.CaptureFilesRequest\x1a%.build.buildgrid.CaptureFilesResponse\"\x00\x12\x81\x01\n\x18GetInstanceNameForRemote\x12\x30.build.buildgrid.GetInstanceNameForRemoteRequest\x1a\x31.build.buildgrid.GetInstanceNameForRemoteResponse\"\x00\x12\x84\x01\n\x19GetInstanceNameForRemotes\x12\x31.build.buildgrid.GetInstanceNameForRemotesRequest\x1a\x32.build.buildgrid.GetInstanceNameForRemotesResponse\"\x00\x12\x8a\x01\n\x1bGetInstanceNameForNamespace\x12\x33.build.buildgrid.GetInstanceNameForNamespaceRequest\x1a\x34.build.buildgrid.GetInstanceNameForNamespaceResponse\"\x00\x12l\n\x11GetLocalDiskUsage\x12).build.buildgrid.GetLocalDiskUsageRequest\x1a*.build.buildgrid.GetLocalDiskUsageResponse\"\x00\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x62uild/buildgrid/local_cas.proto\x12\x0f\x62uild.buildgrid\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x17google/rpc/status.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1egoogle/protobuf/wrappers.proto\"5\n\x1cGetLocalServerDetailsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"Q\n\x12LocalServerDetails\x12\x10\n\x08hostname\x18\x01 \x01(\t\x12\x13\n\x0bprocess_uid\x18\x02 \x01(\x03\x12\x14\n\x0cstorage_root\x18\x03 \x01(\t\"p\n\x18\x46\x65tchMissingBlobsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12=\n\x0c\x62lob_digests\x18\x02 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\xcc\x01\n\x19\x46\x65tchMissingBlobsResponse\x12\x46\n\tresponses\x18\x01 \x03(\x0b\x32\x33.build.buildgrid.FetchMissingBlobsResponse.Response\x1ag\n\x08Response\x12\x37\n\x06\x64igest\x18\x01 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x02 \x01(\x0b\x32\x12.google.rpc.Status\"q\n\x19UploadMissingBlobsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12=\n\x0c\x62lob_digests\x18\x02 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\xce\x01\n\x1aUploadMissingBlobsResponse\x12G\n\tresponses\x18\x01 \x03(\x0b\x32\x34.build.buildgrid.UploadMissingBlobsResponse.Response\x1ag\n\x08Response\x12\x37\n\x06\x64igest\x18\x01 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x02 \x01(\x0b\x32\x12.google.rpc.Status\"\x81\x01\n\x10\x46\x65tchTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12<\n\x0broot_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x18\n\x10\x66\x65tch_file_blobs\x18\x03 \x01(\x08\"\x13\n\x11\x46\x65tchTreeResponse\"h\n\x11UploadTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12<\n\x0broot_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\x14\n\x12UploadTreeResponse\"\xe6\x01\n\x16RemoteApisSocketConfig\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12#\n\x1b\x61\x63tion_cache_update_enabled\x18\x03 \x01(\x08\x12N\n\x0cinstance_map\x18\x04 \x03(\x0b\x32\x38.build.buildgrid.RemoteApisSocketConfig.InstanceMapEntry\x1a\x32\n\x10InstanceMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xc9\x03\n\x10StageTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12<\n\x0broot_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x0c\n\x04path\x18\x03 \x01(\t\x12I\n\x12\x61\x63\x63\x65ss_credentials\x18\x04 \x01(\x0b\x32-.build.buildgrid.StageTreeRequest.Credentials\x12\x43\n\x12remote_apis_socket\x18\x08 \x01(\x0b\x32\'.build.buildgrid.RemoteApisSocketConfig\x12\x1c\n\x14pre_unstage_commands\x18\x05 \x03(\t\x12\x43\n\x0cstaging_mode\x18\x06 \x01(\x0e\x32-.build.buildgrid.StageTreeRequest.StagingMode\x1a\'\n\x0b\x43redentials\x12\x0b\n\x03uid\x18\x01 \x01(\x03\x12\x0b\n\x03gid\x18\x02 \x01(\x03\"6\n\x0bStagingMode\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x08\n\x04\x46USE\x10\x01\x12\x10\n\x0c\x43OPY_OR_LINK\x10\x02\"!\n\x11StageTreeResponse\x12\x0c\n\x04path\x18\x01 \x01(\t\"|\n\x12\x45xportFilesRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12\x41\n\x0coutput_files\x18\x03 \x03(\x0b\x32+.build.bazel.remote.execution.v2.OutputFile\"\x95\x01\n\x13\x45xportFilesResponse\x12@\n\tresponses\x18\x01 \x03(\x0b\x32-.build.buildgrid.ExportFilesResponse.Response\x1a<\n\x08Response\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\"\n\x06status\x18\x02 \x01(\x0b\x32\x12.google.rpc.Status\"\xd9\x02\n\x12\x43\x61ptureTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x06 \x01(\t\x12\x0c\n\x04path\x18\x02 \x03(\t\x12\x1a\n\x12\x62ypass_local_cache\x18\x03 \x01(\x08\x12\x17\n\x0fnode_properties\x18\x04 \x03(\t\x12\x12\n\nmove_files\x18\x05 \x01(\x08\x12_\n\x17output_directory_format\x18\x07 \x01(\x0e\x32>.build.bazel.remote.execution.v2.Command.OutputDirectoryFormat\x12\x13\n\x0bskip_upload\x18\x08 \x01(\x08\x12\x34\n\x0eunix_mode_mask\x18\t \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\x12\x1b\n\x13\x61llow_chmod_to_read\x18\n \x01(\x08\"\xb6\x02\n\x13\x43\x61ptureTreeResponse\x12@\n\tresponses\x18\x01 \x03(\x0b\x32-.build.buildgrid.CaptureTreeResponse.Response\x1a\xdc\x01\n\x08Response\x12\x0c\n\x04path\x18\x01 \x01(\t\x12<\n\x0btree_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x03 \x01(\x0b\x32\x12.google.rpc.Status\x12\x46\n\x15root_directory_digest\x18\x04 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x18\n\x10total_size_bytes\x18\x05 \x01(\x03\"\xb1\x01\n\x10HashFilesRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x03 \x01(\t\x12\x0c\n\x04path\x18\x02 \x03(\t\x12\x17\n\x0fnode_properties\x18\x04 \x03(\t\x12\x34\n\x0eunix_mode_mask\x18\x05 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\x12\x1b\n\x13\x61llow_chmod_to_read\x18\x06 \x01(\x08\"\xac\x02\n\x11HashFilesResponse\x12>\n\tresponses\x18\x01 \x03(\x0b\x32+.build.buildgrid.HashFilesResponse.Response\x1a\xd6\x01\n\x08Response\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x03 \x01(\x0b\x32\x12.google.rpc.Status\x12\x15\n\ris_executable\x18\x04 \x01(\x08\x12H\n\x0fnode_properties\x18\x05 \x01(\x0b\x32/.build.bazel.remote.execution.v2.NodeProperties\"\xf9\x01\n\x13\x43\x61ptureFilesRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x06 \x01(\t\x12\x0c\n\x04path\x18\x02 \x03(\t\x12\x1a\n\x12\x62ypass_local_cache\x18\x03 \x01(\x08\x12\x17\n\x0fnode_properties\x18\x04 \x03(\t\x12\x12\n\nmove_files\x18\x05 \x01(\x08\x12\x13\n\x0bskip_upload\x18\x07 \x01(\x08\x12\x34\n\x0eunix_mode_mask\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\x12\x1b\n\x13\x61llow_chmod_to_read\x18\t \x01(\x08\"\xb8\x02\n\x14\x43\x61ptureFilesResponse\x12\x41\n\tresponses\x18\x01 \x03(\x0b\x32..build.buildgrid.CaptureFilesResponse.Response\x1a\xdc\x01\n\x08Response\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x03 \x01(\x0b\x32\x12.google.rpc.Status\x12\x15\n\ris_executable\x18\x04 \x01(\x08\x12H\n\x0fnode_properties\x18\x06 \x01(\x0b\x32/.build.bazel.remote.execution.v2.NodePropertiesJ\x04\x08\x05\x10\x06\"\x83\x01\n\x1fGetInstanceNameForRemoteRequest\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x15\n\rinstance_name\x18\x02 \x01(\t\x12\x13\n\x0bserver_cert\x18\x03 \x01(\x0c\x12\x12\n\nclient_key\x18\x04 \x01(\x0c\x12\x13\n\x0b\x63lient_cert\x18\x05 \x01(\x0c\"9\n GetInstanceNameForRemoteResponse\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"\x97\x03\n\x06Remote\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x15\n\rinstance_name\x18\x02 \x01(\t\x12\x13\n\x0bserver_cert\x18\x03 \x01(\x0c\x12\x12\n\nclient_key\x18\x04 \x01(\x0c\x12\x13\n\x0b\x63lient_cert\x18\x05 \x01(\x0c\x12\x19\n\x11\x61\x63\x63\x65ss_token_path\x18\x07 \x01(\t\x12?\n\x1c\x61\x63\x63\x65ss_token_reload_interval\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x31\n\x0ekeepalive_time\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bretry_limit\x18\t \x01(\x03\x12.\n\x0bretry_delay\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x32\n\x0frequest_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x11\n\tread_only\x18\x0c \x01(\x08\x12\x10\n\x08protocol\x18\r \x01(\t\"\x81\x02\n GetInstanceNameForRemotesRequest\x12\x15\n\rinstance_name\x18\x03 \x01(\t\x12<\n\x1b\x63ontent_addressable_storage\x18\x01 \x01(\x0b\x32\x17.build.buildgrid.Remote\x12-\n\x0cremote_asset\x18\x02 \x01(\x0b\x32\x17.build.buildgrid.Remote\x12-\n\x0c\x61\x63tion_cache\x18\x04 \x01(\x0b\x32\x17.build.buildgrid.Remote\x12*\n\texecution\x18\x05 \x01(\x0b\x32\x17.build.buildgrid.Remote\":\n!GetInstanceNameForRemotesResponse\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"\x1a\n\x18GetLocalDiskUsageRequest\"D\n\x19GetLocalDiskUsageResponse\x12\x12\n\nsize_bytes\x18\x01 \x01(\x03\x12\x13\n\x0bquota_bytes\x18\x02 \x01(\x03\"\xf6\x01\n\x13NestedServerRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\x12L\n\x12\x61\x63\x63\x65ss_credentials\x18\x03 \x01(\x0b\x32\x30.build.buildgrid.NestedServerRequest.Credentials\x12\x43\n\x12remote_apis_socket\x18\x04 \x01(\x0b\x32\'.build.buildgrid.RemoteApisSocketConfig\x1a\'\n\x0b\x43redentials\x12\x0b\n\x03uid\x18\x01 \x01(\x03\x12\x0b\n\x03gid\x18\x02 \x01(\x03\"\x16\n\x14NestedServerResponse2\xc0\x0b\n\x1eLocalContentAddressableStorage\x12m\n\x15GetLocalServerDetails\x12-.build.buildgrid.GetLocalServerDetailsRequest\x1a#.build.buildgrid.LocalServerDetails\"\x00\x12l\n\x11\x46\x65tchMissingBlobs\x12).build.buildgrid.FetchMissingBlobsRequest\x1a*.build.buildgrid.FetchMissingBlobsResponse\"\x00\x12o\n\x12UploadMissingBlobs\x12*.build.buildgrid.UploadMissingBlobsRequest\x1a+.build.buildgrid.UploadMissingBlobsResponse\"\x00\x12T\n\tFetchTree\x12!.build.buildgrid.FetchTreeRequest\x1a\".build.buildgrid.FetchTreeResponse\"\x00\x12W\n\nUploadTree\x12\".build.buildgrid.UploadTreeRequest\x1a#.build.buildgrid.UploadTreeResponse\"\x00\x12X\n\tStageTree\x12!.build.buildgrid.StageTreeRequest\x1a\".build.buildgrid.StageTreeResponse\"\x00(\x01\x30\x01\x12Z\n\x0b\x45xportFiles\x12#.build.buildgrid.ExportFilesRequest\x1a$.build.buildgrid.ExportFilesResponse\"\x00\x12Z\n\x0b\x43\x61ptureTree\x12#.build.buildgrid.CaptureTreeRequest\x1a$.build.buildgrid.CaptureTreeResponse\"\x00\x12]\n\x0c\x43\x61ptureFiles\x12$.build.buildgrid.CaptureFilesRequest\x1a%.build.buildgrid.CaptureFilesResponse\"\x00\x12T\n\tHashFiles\x12!.build.buildgrid.HashFilesRequest\x1a\".build.buildgrid.HashFilesResponse\"\x00\x12\x81\x01\n\x18GetInstanceNameForRemote\x12\x30.build.buildgrid.GetInstanceNameForRemoteRequest\x1a\x31.build.buildgrid.GetInstanceNameForRemoteResponse\"\x00\x12\x84\x01\n\x19GetInstanceNameForRemotes\x12\x31.build.buildgrid.GetInstanceNameForRemotesRequest\x1a\x32.build.buildgrid.GetInstanceNameForRemotesResponse\"\x00\x12l\n\x11GetLocalDiskUsage\x12).build.buildgrid.GetLocalDiskUsageRequest\x1a*.build.buildgrid.GetLocalDiskUsageResponse\"\x00\x12\x61\n\x0cNestedServer\x12$.build.buildgrid.NestedServerRequest\x1a%.build.buildgrid.NestedServerResponse\"\x00(\x01\x30\x01\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'build.buildgrid.local_cas_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None + _globals['_REMOTEAPISSOCKETCONFIG_INSTANCEMAPENTRY']._loaded_options = None + _globals['_REMOTEAPISSOCKETCONFIG_INSTANCEMAPENTRY']._serialized_options = b'8\001' _globals['_GETLOCALSERVERDETAILSREQUEST']._serialized_start=197 _globals['_GETLOCALSERVERDETAILSREQUEST']._serialized_end=250 _globals['_LOCALSERVERDETAILS']._serialized_start=252 @@ -59,44 +61,62 @@ _globals['_UPLOADTREEREQUEST']._serialized_end=1237 _globals['_UPLOADTREERESPONSE']._serialized_start=1239 _globals['_UPLOADTREERESPONSE']._serialized_end=1259 - _globals['_STAGETREEREQUEST']._serialized_start=1262 - _globals['_STAGETREEREQUEST']._serialized_end=1683 - _globals['_STAGETREEREQUEST_CREDENTIALS']._serialized_start=1588 - _globals['_STAGETREEREQUEST_CREDENTIALS']._serialized_end=1627 - _globals['_STAGETREEREQUEST_STAGINGMODE']._serialized_start=1629 - _globals['_STAGETREEREQUEST_STAGINGMODE']._serialized_end=1683 - _globals['_STAGETREERESPONSE']._serialized_start=1685 - _globals['_STAGETREERESPONSE']._serialized_end=1718 - _globals['_CAPTURETREEREQUEST']._serialized_start=1721 - _globals['_CAPTURETREEREQUEST']._serialized_end=2037 - _globals['_CAPTURETREERESPONSE']._serialized_start=2040 - _globals['_CAPTURETREERESPONSE']._serialized_end=2324 - _globals['_CAPTURETREERESPONSE_RESPONSE']._serialized_start=2130 - _globals['_CAPTURETREERESPONSE_RESPONSE']._serialized_end=2324 - _globals['_CAPTUREFILESREQUEST']._serialized_start=2327 - _globals['_CAPTUREFILESREQUEST']._serialized_end=2547 - _globals['_CAPTUREFILESRESPONSE']._serialized_start=2550 - _globals['_CAPTUREFILESRESPONSE']._serialized_end=2862 - _globals['_CAPTUREFILESRESPONSE_RESPONSE']._serialized_start=2642 - _globals['_CAPTUREFILESRESPONSE_RESPONSE']._serialized_end=2862 - _globals['_GETINSTANCENAMEFORREMOTEREQUEST']._serialized_start=2865 - _globals['_GETINSTANCENAMEFORREMOTEREQUEST']._serialized_end=2996 - _globals['_GETINSTANCENAMEFORREMOTERESPONSE']._serialized_start=2998 - _globals['_GETINSTANCENAMEFORREMOTERESPONSE']._serialized_end=3055 - _globals['_REMOTE']._serialized_start=3058 - _globals['_REMOTE']._serialized_end=3447 - _globals['_GETINSTANCENAMEFORREMOTESREQUEST']._serialized_start=3450 - _globals['_GETINSTANCENAMEFORREMOTESREQUEST']._serialized_end=3707 - _globals['_GETINSTANCENAMEFORREMOTESRESPONSE']._serialized_start=3709 - _globals['_GETINSTANCENAMEFORREMOTESRESPONSE']._serialized_end=3767 - _globals['_GETINSTANCENAMEFORNAMESPACEREQUEST']._serialized_start=3769 - _globals['_GETINSTANCENAMEFORNAMESPACEREQUEST']._serialized_end=3842 - _globals['_GETINSTANCENAMEFORNAMESPACERESPONSE']._serialized_start=3844 - _globals['_GETINSTANCENAMEFORNAMESPACERESPONSE']._serialized_end=3904 - _globals['_GETLOCALDISKUSAGEREQUEST']._serialized_start=3906 - _globals['_GETLOCALDISKUSAGEREQUEST']._serialized_end=3932 - _globals['_GETLOCALDISKUSAGERESPONSE']._serialized_start=3934 - _globals['_GETLOCALDISKUSAGERESPONSE']._serialized_end=4002 - _globals['_LOCALCONTENTADDRESSABLESTORAGE']._serialized_start=4005 - _globals['_LOCALCONTENTADDRESSABLESTORAGE']._serialized_end=5341 + _globals['_REMOTEAPISSOCKETCONFIG']._serialized_start=1262 + _globals['_REMOTEAPISSOCKETCONFIG']._serialized_end=1492 + _globals['_REMOTEAPISSOCKETCONFIG_INSTANCEMAPENTRY']._serialized_start=1442 + _globals['_REMOTEAPISSOCKETCONFIG_INSTANCEMAPENTRY']._serialized_end=1492 + _globals['_STAGETREEREQUEST']._serialized_start=1495 + _globals['_STAGETREEREQUEST']._serialized_end=1952 + _globals['_STAGETREEREQUEST_CREDENTIALS']._serialized_start=1857 + _globals['_STAGETREEREQUEST_CREDENTIALS']._serialized_end=1896 + _globals['_STAGETREEREQUEST_STAGINGMODE']._serialized_start=1898 + _globals['_STAGETREEREQUEST_STAGINGMODE']._serialized_end=1952 + _globals['_STAGETREERESPONSE']._serialized_start=1954 + _globals['_STAGETREERESPONSE']._serialized_end=1987 + _globals['_EXPORTFILESREQUEST']._serialized_start=1989 + _globals['_EXPORTFILESREQUEST']._serialized_end=2113 + _globals['_EXPORTFILESRESPONSE']._serialized_start=2116 + _globals['_EXPORTFILESRESPONSE']._serialized_end=2265 + _globals['_EXPORTFILESRESPONSE_RESPONSE']._serialized_start=2205 + _globals['_EXPORTFILESRESPONSE_RESPONSE']._serialized_end=2265 + _globals['_CAPTURETREEREQUEST']._serialized_start=2268 + _globals['_CAPTURETREEREQUEST']._serialized_end=2613 + _globals['_CAPTURETREERESPONSE']._serialized_start=2616 + _globals['_CAPTURETREERESPONSE']._serialized_end=2926 + _globals['_CAPTURETREERESPONSE_RESPONSE']._serialized_start=2706 + _globals['_CAPTURETREERESPONSE_RESPONSE']._serialized_end=2926 + _globals['_HASHFILESREQUEST']._serialized_start=2929 + _globals['_HASHFILESREQUEST']._serialized_end=3106 + _globals['_HASHFILESRESPONSE']._serialized_start=3109 + _globals['_HASHFILESRESPONSE']._serialized_end=3409 + _globals['_HASHFILESRESPONSE_RESPONSE']._serialized_start=3195 + _globals['_HASHFILESRESPONSE_RESPONSE']._serialized_end=3409 + _globals['_CAPTUREFILESREQUEST']._serialized_start=3412 + _globals['_CAPTUREFILESREQUEST']._serialized_end=3661 + _globals['_CAPTUREFILESRESPONSE']._serialized_start=3664 + _globals['_CAPTUREFILESRESPONSE']._serialized_end=3976 + _globals['_CAPTUREFILESRESPONSE_RESPONSE']._serialized_start=3756 + _globals['_CAPTUREFILESRESPONSE_RESPONSE']._serialized_end=3976 + _globals['_GETINSTANCENAMEFORREMOTEREQUEST']._serialized_start=3979 + _globals['_GETINSTANCENAMEFORREMOTEREQUEST']._serialized_end=4110 + _globals['_GETINSTANCENAMEFORREMOTERESPONSE']._serialized_start=4112 + _globals['_GETINSTANCENAMEFORREMOTERESPONSE']._serialized_end=4169 + _globals['_REMOTE']._serialized_start=4172 + _globals['_REMOTE']._serialized_end=4579 + _globals['_GETINSTANCENAMEFORREMOTESREQUEST']._serialized_start=4582 + _globals['_GETINSTANCENAMEFORREMOTESREQUEST']._serialized_end=4839 + _globals['_GETINSTANCENAMEFORREMOTESRESPONSE']._serialized_start=4841 + _globals['_GETINSTANCENAMEFORREMOTESRESPONSE']._serialized_end=4899 + _globals['_GETLOCALDISKUSAGEREQUEST']._serialized_start=4901 + _globals['_GETLOCALDISKUSAGEREQUEST']._serialized_end=4927 + _globals['_GETLOCALDISKUSAGERESPONSE']._serialized_start=4929 + _globals['_GETLOCALDISKUSAGERESPONSE']._serialized_end=4997 + _globals['_NESTEDSERVERREQUEST']._serialized_start=5000 + _globals['_NESTEDSERVERREQUEST']._serialized_end=5246 + _globals['_NESTEDSERVERREQUEST_CREDENTIALS']._serialized_start=1857 + _globals['_NESTEDSERVERREQUEST_CREDENTIALS']._serialized_end=1896 + _globals['_NESTEDSERVERRESPONSE']._serialized_start=5248 + _globals['_NESTEDSERVERRESPONSE']._serialized_end=5270 + _globals['_LOCALCONTENTADDRESSABLESTORAGE']._serialized_start=5273 + _globals['_LOCALCONTENTADDRESSABLESTORAGE']._serialized_end=6745 # @@protoc_insertion_point(module_scope) diff --git a/src/buildstream/_protos/build/buildgrid/local_cas_pb2.pyi b/src/buildstream/_protos/build/buildgrid/local_cas_pb2.pyi index 86a3bf22a..d4b7b8845 100644 --- a/src/buildstream/_protos/build/buildgrid/local_cas_pb2.pyi +++ b/src/buildstream/_protos/build/buildgrid/local_cas_pb2.pyi @@ -94,8 +94,27 @@ class UploadTreeResponse(_message.Message): __slots__ = () def __init__(self) -> None: ... +class RemoteApisSocketConfig(_message.Message): + __slots__ = ("instance_name", "path", "action_cache_update_enabled", "instance_map") + class InstanceMapEntry(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] + PATH_FIELD_NUMBER: _ClassVar[int] + ACTION_CACHE_UPDATE_ENABLED_FIELD_NUMBER: _ClassVar[int] + INSTANCE_MAP_FIELD_NUMBER: _ClassVar[int] + instance_name: str + path: str + action_cache_update_enabled: bool + instance_map: _containers.ScalarMap[str, str] + def __init__(self, instance_name: _Optional[str] = ..., path: _Optional[str] = ..., action_cache_update_enabled: bool = ..., instance_map: _Optional[_Mapping[str, str]] = ...) -> None: ... + class StageTreeRequest(_message.Message): - __slots__ = ("instance_name", "root_digest", "path", "access_credentials", "remote_apis_socket_path", "pre_unstage_commands", "staging_mode") + __slots__ = ("instance_name", "root_digest", "path", "access_credentials", "remote_apis_socket", "pre_unstage_commands", "staging_mode") class StagingMode(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () DEFAULT: _ClassVar[StageTreeRequest.StagingMode] @@ -115,17 +134,17 @@ class StageTreeRequest(_message.Message): ROOT_DIGEST_FIELD_NUMBER: _ClassVar[int] PATH_FIELD_NUMBER: _ClassVar[int] ACCESS_CREDENTIALS_FIELD_NUMBER: _ClassVar[int] - REMOTE_APIS_SOCKET_PATH_FIELD_NUMBER: _ClassVar[int] + REMOTE_APIS_SOCKET_FIELD_NUMBER: _ClassVar[int] PRE_UNSTAGE_COMMANDS_FIELD_NUMBER: _ClassVar[int] STAGING_MODE_FIELD_NUMBER: _ClassVar[int] instance_name: str root_digest: _remote_execution_pb2.Digest path: str access_credentials: StageTreeRequest.Credentials - remote_apis_socket_path: str + remote_apis_socket: RemoteApisSocketConfig pre_unstage_commands: _containers.RepeatedScalarFieldContainer[str] staging_mode: StageTreeRequest.StagingMode - def __init__(self, instance_name: _Optional[str] = ..., root_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., path: _Optional[str] = ..., access_credentials: _Optional[_Union[StageTreeRequest.Credentials, _Mapping]] = ..., remote_apis_socket_path: _Optional[str] = ..., pre_unstage_commands: _Optional[_Iterable[str]] = ..., staging_mode: _Optional[_Union[StageTreeRequest.StagingMode, str]] = ...) -> None: ... + def __init__(self, instance_name: _Optional[str] = ..., root_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., path: _Optional[str] = ..., access_credentials: _Optional[_Union[StageTreeRequest.Credentials, _Mapping]] = ..., remote_apis_socket: _Optional[_Union[RemoteApisSocketConfig, _Mapping]] = ..., pre_unstage_commands: _Optional[_Iterable[str]] = ..., staging_mode: _Optional[_Union[StageTreeRequest.StagingMode, str]] = ...) -> None: ... class StageTreeResponse(_message.Message): __slots__ = ("path",) @@ -133,8 +152,31 @@ class StageTreeResponse(_message.Message): path: str def __init__(self, path: _Optional[str] = ...) -> None: ... +class ExportFilesRequest(_message.Message): + __slots__ = ("instance_name", "path", "output_files") + INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] + PATH_FIELD_NUMBER: _ClassVar[int] + OUTPUT_FILES_FIELD_NUMBER: _ClassVar[int] + instance_name: str + path: str + output_files: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.OutputFile] + def __init__(self, instance_name: _Optional[str] = ..., path: _Optional[str] = ..., output_files: _Optional[_Iterable[_Union[_remote_execution_pb2.OutputFile, _Mapping]]] = ...) -> None: ... + +class ExportFilesResponse(_message.Message): + __slots__ = ("responses",) + class Response(_message.Message): + __slots__ = ("path", "status") + PATH_FIELD_NUMBER: _ClassVar[int] + STATUS_FIELD_NUMBER: _ClassVar[int] + path: str + status: _status_pb2.Status + def __init__(self, path: _Optional[str] = ..., status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ...) -> None: ... + RESPONSES_FIELD_NUMBER: _ClassVar[int] + responses: _containers.RepeatedCompositeFieldContainer[ExportFilesResponse.Response] + def __init__(self, responses: _Optional[_Iterable[_Union[ExportFilesResponse.Response, _Mapping]]] = ...) -> None: ... + class CaptureTreeRequest(_message.Message): - __slots__ = ("instance_name", "root", "path", "bypass_local_cache", "node_properties", "move_files", "output_directory_format", "skip_upload", "unix_mode_mask") + __slots__ = ("instance_name", "root", "path", "bypass_local_cache", "node_properties", "move_files", "output_directory_format", "skip_upload", "unix_mode_mask", "allow_chmod_to_read") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] ROOT_FIELD_NUMBER: _ClassVar[int] PATH_FIELD_NUMBER: _ClassVar[int] @@ -144,6 +186,7 @@ class CaptureTreeRequest(_message.Message): OUTPUT_DIRECTORY_FORMAT_FIELD_NUMBER: _ClassVar[int] SKIP_UPLOAD_FIELD_NUMBER: _ClassVar[int] UNIX_MODE_MASK_FIELD_NUMBER: _ClassVar[int] + ALLOW_CHMOD_TO_READ_FIELD_NUMBER: _ClassVar[int] instance_name: str root: str path: _containers.RepeatedScalarFieldContainer[str] @@ -153,27 +196,65 @@ class CaptureTreeRequest(_message.Message): output_directory_format: _remote_execution_pb2.Command.OutputDirectoryFormat skip_upload: bool unix_mode_mask: _wrappers_pb2.UInt32Value - def __init__(self, instance_name: _Optional[str] = ..., root: _Optional[str] = ..., path: _Optional[_Iterable[str]] = ..., bypass_local_cache: bool = ..., node_properties: _Optional[_Iterable[str]] = ..., move_files: bool = ..., output_directory_format: _Optional[_Union[_remote_execution_pb2.Command.OutputDirectoryFormat, str]] = ..., skip_upload: bool = ..., unix_mode_mask: _Optional[_Union[_wrappers_pb2.UInt32Value, _Mapping]] = ...) -> None: ... + allow_chmod_to_read: bool + def __init__(self, instance_name: _Optional[str] = ..., root: _Optional[str] = ..., path: _Optional[_Iterable[str]] = ..., bypass_local_cache: bool = ..., node_properties: _Optional[_Iterable[str]] = ..., move_files: bool = ..., output_directory_format: _Optional[_Union[_remote_execution_pb2.Command.OutputDirectoryFormat, str]] = ..., skip_upload: bool = ..., unix_mode_mask: _Optional[_Union[_wrappers_pb2.UInt32Value, _Mapping]] = ..., allow_chmod_to_read: bool = ...) -> None: ... class CaptureTreeResponse(_message.Message): __slots__ = ("responses",) class Response(_message.Message): - __slots__ = ("path", "tree_digest", "status", "root_directory_digest") + __slots__ = ("path", "tree_digest", "status", "root_directory_digest", "total_size_bytes") PATH_FIELD_NUMBER: _ClassVar[int] TREE_DIGEST_FIELD_NUMBER: _ClassVar[int] STATUS_FIELD_NUMBER: _ClassVar[int] ROOT_DIRECTORY_DIGEST_FIELD_NUMBER: _ClassVar[int] + TOTAL_SIZE_BYTES_FIELD_NUMBER: _ClassVar[int] path: str tree_digest: _remote_execution_pb2.Digest status: _status_pb2.Status root_directory_digest: _remote_execution_pb2.Digest - def __init__(self, path: _Optional[str] = ..., tree_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ..., root_directory_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ...) -> None: ... + total_size_bytes: int + def __init__(self, path: _Optional[str] = ..., tree_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ..., root_directory_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., total_size_bytes: _Optional[int] = ...) -> None: ... RESPONSES_FIELD_NUMBER: _ClassVar[int] responses: _containers.RepeatedCompositeFieldContainer[CaptureTreeResponse.Response] def __init__(self, responses: _Optional[_Iterable[_Union[CaptureTreeResponse.Response, _Mapping]]] = ...) -> None: ... +class HashFilesRequest(_message.Message): + __slots__ = ("instance_name", "root", "path", "node_properties", "unix_mode_mask", "allow_chmod_to_read") + INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] + ROOT_FIELD_NUMBER: _ClassVar[int] + PATH_FIELD_NUMBER: _ClassVar[int] + NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] + UNIX_MODE_MASK_FIELD_NUMBER: _ClassVar[int] + ALLOW_CHMOD_TO_READ_FIELD_NUMBER: _ClassVar[int] + instance_name: str + root: str + path: _containers.RepeatedScalarFieldContainer[str] + node_properties: _containers.RepeatedScalarFieldContainer[str] + unix_mode_mask: _wrappers_pb2.UInt32Value + allow_chmod_to_read: bool + def __init__(self, instance_name: _Optional[str] = ..., root: _Optional[str] = ..., path: _Optional[_Iterable[str]] = ..., node_properties: _Optional[_Iterable[str]] = ..., unix_mode_mask: _Optional[_Union[_wrappers_pb2.UInt32Value, _Mapping]] = ..., allow_chmod_to_read: bool = ...) -> None: ... + +class HashFilesResponse(_message.Message): + __slots__ = ("responses",) + class Response(_message.Message): + __slots__ = ("path", "digest", "status", "is_executable", "node_properties") + PATH_FIELD_NUMBER: _ClassVar[int] + DIGEST_FIELD_NUMBER: _ClassVar[int] + STATUS_FIELD_NUMBER: _ClassVar[int] + IS_EXECUTABLE_FIELD_NUMBER: _ClassVar[int] + NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] + path: str + digest: _remote_execution_pb2.Digest + status: _status_pb2.Status + is_executable: bool + node_properties: _remote_execution_pb2.NodeProperties + def __init__(self, path: _Optional[str] = ..., digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ..., is_executable: bool = ..., node_properties: _Optional[_Union[_remote_execution_pb2.NodeProperties, _Mapping]] = ...) -> None: ... + RESPONSES_FIELD_NUMBER: _ClassVar[int] + responses: _containers.RepeatedCompositeFieldContainer[HashFilesResponse.Response] + def __init__(self, responses: _Optional[_Iterable[_Union[HashFilesResponse.Response, _Mapping]]] = ...) -> None: ... + class CaptureFilesRequest(_message.Message): - __slots__ = ("instance_name", "root", "path", "bypass_local_cache", "node_properties", "move_files", "skip_upload", "unix_mode_mask") + __slots__ = ("instance_name", "root", "path", "bypass_local_cache", "node_properties", "move_files", "skip_upload", "unix_mode_mask", "allow_chmod_to_read") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] ROOT_FIELD_NUMBER: _ClassVar[int] PATH_FIELD_NUMBER: _ClassVar[int] @@ -182,6 +263,7 @@ class CaptureFilesRequest(_message.Message): MOVE_FILES_FIELD_NUMBER: _ClassVar[int] SKIP_UPLOAD_FIELD_NUMBER: _ClassVar[int] UNIX_MODE_MASK_FIELD_NUMBER: _ClassVar[int] + ALLOW_CHMOD_TO_READ_FIELD_NUMBER: _ClassVar[int] instance_name: str root: str path: _containers.RepeatedScalarFieldContainer[str] @@ -190,7 +272,8 @@ class CaptureFilesRequest(_message.Message): move_files: bool skip_upload: bool unix_mode_mask: _wrappers_pb2.UInt32Value - def __init__(self, instance_name: _Optional[str] = ..., root: _Optional[str] = ..., path: _Optional[_Iterable[str]] = ..., bypass_local_cache: bool = ..., node_properties: _Optional[_Iterable[str]] = ..., move_files: bool = ..., skip_upload: bool = ..., unix_mode_mask: _Optional[_Union[_wrappers_pb2.UInt32Value, _Mapping]] = ...) -> None: ... + allow_chmod_to_read: bool + def __init__(self, instance_name: _Optional[str] = ..., root: _Optional[str] = ..., path: _Optional[_Iterable[str]] = ..., bypass_local_cache: bool = ..., node_properties: _Optional[_Iterable[str]] = ..., move_files: bool = ..., skip_upload: bool = ..., unix_mode_mask: _Optional[_Union[_wrappers_pb2.UInt32Value, _Mapping]] = ..., allow_chmod_to_read: bool = ...) -> None: ... class CaptureFilesResponse(_message.Message): __slots__ = ("responses",) @@ -232,7 +315,7 @@ class GetInstanceNameForRemoteResponse(_message.Message): def __init__(self, instance_name: _Optional[str] = ...) -> None: ... class Remote(_message.Message): - __slots__ = ("url", "instance_name", "server_cert", "client_key", "client_cert", "access_token_path", "access_token_reload_interval", "keepalive_time", "retry_limit", "retry_delay", "request_timeout", "read_only") + __slots__ = ("url", "instance_name", "server_cert", "client_key", "client_cert", "access_token_path", "access_token_reload_interval", "keepalive_time", "retry_limit", "retry_delay", "request_timeout", "read_only", "protocol") URL_FIELD_NUMBER: _ClassVar[int] INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] SERVER_CERT_FIELD_NUMBER: _ClassVar[int] @@ -245,6 +328,7 @@ class Remote(_message.Message): RETRY_DELAY_FIELD_NUMBER: _ClassVar[int] REQUEST_TIMEOUT_FIELD_NUMBER: _ClassVar[int] READ_ONLY_FIELD_NUMBER: _ClassVar[int] + PROTOCOL_FIELD_NUMBER: _ClassVar[int] url: str instance_name: str server_cert: bytes @@ -257,7 +341,8 @@ class Remote(_message.Message): retry_delay: _duration_pb2.Duration request_timeout: _duration_pb2.Duration read_only: bool - def __init__(self, url: _Optional[str] = ..., instance_name: _Optional[str] = ..., server_cert: _Optional[bytes] = ..., client_key: _Optional[bytes] = ..., client_cert: _Optional[bytes] = ..., access_token_path: _Optional[str] = ..., access_token_reload_interval: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., keepalive_time: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., retry_limit: _Optional[int] = ..., retry_delay: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., request_timeout: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., read_only: bool = ...) -> None: ... + protocol: str + def __init__(self, url: _Optional[str] = ..., instance_name: _Optional[str] = ..., server_cert: _Optional[bytes] = ..., client_key: _Optional[bytes] = ..., client_cert: _Optional[bytes] = ..., access_token_path: _Optional[str] = ..., access_token_reload_interval: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., keepalive_time: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., retry_limit: _Optional[int] = ..., retry_delay: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., request_timeout: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., read_only: bool = ..., protocol: _Optional[str] = ...) -> None: ... class GetInstanceNameForRemotesRequest(_message.Message): __slots__ = ("instance_name", "content_addressable_storage", "remote_asset", "action_cache", "execution") @@ -279,20 +364,6 @@ class GetInstanceNameForRemotesResponse(_message.Message): instance_name: str def __init__(self, instance_name: _Optional[str] = ...) -> None: ... -class GetInstanceNameForNamespaceRequest(_message.Message): - __slots__ = ("instance_name", "root") - INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] - ROOT_FIELD_NUMBER: _ClassVar[int] - instance_name: str - root: str - def __init__(self, instance_name: _Optional[str] = ..., root: _Optional[str] = ...) -> None: ... - -class GetInstanceNameForNamespaceResponse(_message.Message): - __slots__ = ("instance_name",) - INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] - instance_name: str - def __init__(self, instance_name: _Optional[str] = ...) -> None: ... - class GetLocalDiskUsageRequest(_message.Message): __slots__ = () def __init__(self) -> None: ... @@ -304,3 +375,26 @@ class GetLocalDiskUsageResponse(_message.Message): size_bytes: int quota_bytes: int def __init__(self, size_bytes: _Optional[int] = ..., quota_bytes: _Optional[int] = ...) -> None: ... + +class NestedServerRequest(_message.Message): + __slots__ = ("instance_name", "path", "access_credentials", "remote_apis_socket") + class Credentials(_message.Message): + __slots__ = ("uid", "gid") + UID_FIELD_NUMBER: _ClassVar[int] + GID_FIELD_NUMBER: _ClassVar[int] + uid: int + gid: int + def __init__(self, uid: _Optional[int] = ..., gid: _Optional[int] = ...) -> None: ... + INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] + PATH_FIELD_NUMBER: _ClassVar[int] + ACCESS_CREDENTIALS_FIELD_NUMBER: _ClassVar[int] + REMOTE_APIS_SOCKET_FIELD_NUMBER: _ClassVar[int] + instance_name: str + path: str + access_credentials: NestedServerRequest.Credentials + remote_apis_socket: RemoteApisSocketConfig + def __init__(self, instance_name: _Optional[str] = ..., path: _Optional[str] = ..., access_credentials: _Optional[_Union[NestedServerRequest.Credentials, _Mapping]] = ..., remote_apis_socket: _Optional[_Union[RemoteApisSocketConfig, _Mapping]] = ...) -> None: ... + +class NestedServerResponse(_message.Message): + __slots__ = () + def __init__(self) -> None: ... diff --git a/src/buildstream/_protos/build/buildgrid/local_cas_pb2_grpc.py b/src/buildstream/_protos/build/buildgrid/local_cas_pb2_grpc.py index 6c0a69e7e..a66005009 100644 --- a/src/buildstream/_protos/build/buildgrid/local_cas_pb2_grpc.py +++ b/src/buildstream/_protos/build/buildgrid/local_cas_pb2_grpc.py @@ -64,6 +64,11 @@ def __init__(self, channel): request_serializer=build_dot_buildgrid_dot_local__cas__pb2.StageTreeRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.StageTreeResponse.FromString, _registered_method=True) + self.ExportFiles = channel.unary_unary( + '/build.buildgrid.LocalContentAddressableStorage/ExportFiles', + request_serializer=build_dot_buildgrid_dot_local__cas__pb2.ExportFilesRequest.SerializeToString, + response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.ExportFilesResponse.FromString, + _registered_method=True) self.CaptureTree = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/CaptureTree', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureTreeRequest.SerializeToString, @@ -74,6 +79,11 @@ def __init__(self, channel): request_serializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureFilesRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureFilesResponse.FromString, _registered_method=True) + self.HashFiles = channel.unary_unary( + '/build.buildgrid.LocalContentAddressableStorage/HashFiles', + request_serializer=build_dot_buildgrid_dot_local__cas__pb2.HashFilesRequest.SerializeToString, + response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.HashFilesResponse.FromString, + _registered_method=True) self.GetInstanceNameForRemote = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/GetInstanceNameForRemote', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemoteRequest.SerializeToString, @@ -84,16 +94,16 @@ def __init__(self, channel): request_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemotesRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemotesResponse.FromString, _registered_method=True) - self.GetInstanceNameForNamespace = channel.unary_unary( - '/build.buildgrid.LocalContentAddressableStorage/GetInstanceNameForNamespace', - request_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceRequest.SerializeToString, - response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceResponse.FromString, - _registered_method=True) self.GetLocalDiskUsage = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/GetLocalDiskUsage', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageResponse.FromString, _registered_method=True) + self.NestedServer = channel.stream_stream( + '/build.buildgrid.LocalContentAddressableStorage/NestedServer', + request_serializer=build_dot_buildgrid_dot_local__cas__pb2.NestedServerRequest.SerializeToString, + response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.NestedServerResponse.FromString, + _registered_method=True) class LocalContentAddressableStorageServicer(object): @@ -201,6 +211,25 @@ def StageTree(self, request_iterator, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ExportFiles(self, request, context): + """Export files to a user specified location on the local filesystem. + + The way in which files are populated is implementation defined, + however, the server will guarantee that mutations to these files + will not corrupt the local cache. + + This does not create any directories. The client needs to ensure that + the parent directory of each output file exists and is writable. + + Missing blobs are fetched, if a CAS remote is configured. + + Errors: + * `FAILED_PRECONDITION`: The destination directory does not exist. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def CaptureTree(self, request, context): """Capture a directory tree from the local filesystem. @@ -247,6 +276,21 @@ def CaptureFiles(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def HashFiles(self, request, context): + """Hash files from the local filesystem. + + This asks the local CAS server to hash the file on behalf of the + client and populate the inode cache for efficent subqeuent hashing + of the same file. + + Unlike CaptureFiles which always store the file content either to + local storage or remote CAS or both. HashFiles only return the hash + and populate the inode cache. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def GetInstanceNameForRemote(self, request, context): """Configure remote CAS endpoint. @@ -270,19 +314,23 @@ def GetInstanceNameForRemotes(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def GetInstanceNameForNamespace(self, request, context): - """Configure sandboxed clients. - - This returns a string that can be used as instance_name to access - this service from clients running in the specified filesystem/mount - namespace or chroot environment + def GetLocalDiskUsage(self, request, context): + """Query total space used by the local cache. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def GetLocalDiskUsage(self, request, context): - """Query total space used by the local cache. + def NestedServer(self, request_iterator, context): + """Create a nested server restricted to a directory tree in the local filesystem. + + The nested server is started when the server receives the initial request + and it is ready to be used on the initial (non-error) response from the + server. + + The nested server will shut down when the server either receives an + additional request (with all fields unset) or when the stream is closed. + The server will send an additional response after cleanup is complete. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -321,6 +369,11 @@ def add_LocalContentAddressableStorageServicer_to_server(servicer, server): request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.StageTreeRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.StageTreeResponse.SerializeToString, ), + 'ExportFiles': grpc.unary_unary_rpc_method_handler( + servicer.ExportFiles, + request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.ExportFilesRequest.FromString, + response_serializer=build_dot_buildgrid_dot_local__cas__pb2.ExportFilesResponse.SerializeToString, + ), 'CaptureTree': grpc.unary_unary_rpc_method_handler( servicer.CaptureTree, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureTreeRequest.FromString, @@ -331,6 +384,11 @@ def add_LocalContentAddressableStorageServicer_to_server(servicer, server): request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureFilesRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureFilesResponse.SerializeToString, ), + 'HashFiles': grpc.unary_unary_rpc_method_handler( + servicer.HashFiles, + request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.HashFilesRequest.FromString, + response_serializer=build_dot_buildgrid_dot_local__cas__pb2.HashFilesResponse.SerializeToString, + ), 'GetInstanceNameForRemote': grpc.unary_unary_rpc_method_handler( servicer.GetInstanceNameForRemote, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemoteRequest.FromString, @@ -341,16 +399,16 @@ def add_LocalContentAddressableStorageServicer_to_server(servicer, server): request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemotesRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemotesResponse.SerializeToString, ), - 'GetInstanceNameForNamespace': grpc.unary_unary_rpc_method_handler( - servicer.GetInstanceNameForNamespace, - request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceRequest.FromString, - response_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceResponse.SerializeToString, - ), 'GetLocalDiskUsage': grpc.unary_unary_rpc_method_handler( servicer.GetLocalDiskUsage, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageResponse.SerializeToString, ), + 'NestedServer': grpc.stream_stream_rpc_method_handler( + servicer.NestedServer, + request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.NestedServerRequest.FromString, + response_serializer=build_dot_buildgrid_dot_local__cas__pb2.NestedServerResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'build.buildgrid.LocalContentAddressableStorage', rpc_method_handlers) @@ -524,6 +582,33 @@ def StageTree(request_iterator, metadata, _registered_method=True) + @staticmethod + def ExportFiles(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/build.buildgrid.LocalContentAddressableStorage/ExportFiles', + build_dot_buildgrid_dot_local__cas__pb2.ExportFilesRequest.SerializeToString, + build_dot_buildgrid_dot_local__cas__pb2.ExportFilesResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def CaptureTree(request, target, @@ -578,6 +663,33 @@ def CaptureFiles(request, metadata, _registered_method=True) + @staticmethod + def HashFiles(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/build.buildgrid.LocalContentAddressableStorage/HashFiles', + build_dot_buildgrid_dot_local__cas__pb2.HashFilesRequest.SerializeToString, + build_dot_buildgrid_dot_local__cas__pb2.HashFilesResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def GetInstanceNameForRemote(request, target, @@ -633,7 +745,7 @@ def GetInstanceNameForRemotes(request, _registered_method=True) @staticmethod - def GetInstanceNameForNamespace(request, + def GetLocalDiskUsage(request, target, options=(), channel_credentials=None, @@ -646,9 +758,9 @@ def GetInstanceNameForNamespace(request, return grpc.experimental.unary_unary( request, target, - '/build.buildgrid.LocalContentAddressableStorage/GetInstanceNameForNamespace', - build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceRequest.SerializeToString, - build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceResponse.FromString, + '/build.buildgrid.LocalContentAddressableStorage/GetLocalDiskUsage', + build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageRequest.SerializeToString, + build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageResponse.FromString, options, channel_credentials, insecure, @@ -660,7 +772,7 @@ def GetInstanceNameForNamespace(request, _registered_method=True) @staticmethod - def GetLocalDiskUsage(request, + def NestedServer(request_iterator, target, options=(), channel_credentials=None, @@ -670,12 +782,12 @@ def GetLocalDiskUsage(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary( - request, + return grpc.experimental.stream_stream( + request_iterator, target, - '/build.buildgrid.LocalContentAddressableStorage/GetLocalDiskUsage', - build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageRequest.SerializeToString, - build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageResponse.FromString, + '/build.buildgrid.LocalContentAddressableStorage/NestedServer', + build_dot_buildgrid_dot_local__cas__pb2.NestedServerRequest.SerializeToString, + build_dot_buildgrid_dot_local__cas__pb2.NestedServerResponse.FromString, options, channel_credentials, insecure, diff --git a/src/buildstream/_remotespec.py b/src/buildstream/_remotespec.py index f0ffd4d3e..375968ff6 100644 --- a/src/buildstream/_remotespec.py +++ b/src/buildstream/_remotespec.py @@ -65,6 +65,7 @@ def __init__( url: str, *, push: bool = False, + protocol: Optional[str] = None, server_cert: Optional[str] = None, client_key: Optional[str] = None, client_cert: Optional[str] = None, @@ -75,6 +76,8 @@ def __init__( spec_node: Optional[MappingNode] = None, ) -> None: + provenance = "{}: ".format(spec_node.get_provenance()) if spec_node else "" + # # Public members # @@ -88,6 +91,19 @@ def __init__( # The url of the remote, this may contain a port number self.url: str = url + # The protocol to use + self.protocol: str = protocol or "grpc" + if self.protocol not in ["grpc", "http"]: + raise RemoteError(f"{provenance}Value for 'protocol' must be 'grpc' or 'http', got: {self.protocol}") + + if self.protocol == "http": + if self.remote_type != RemoteType.STORAGE: + raise RemoteError( + f"{provenance}Value for remote 'type' must be '{RemoteType.STORAGE}' for HTTP remotes" + ) + if self.push: + raise RemoteError(f"{provenance}Push is not supported for HTTP remotes") + # The name of the grpc service to talk to at this remote url self.instance_name: Optional[str] = instance_name @@ -127,6 +143,7 @@ def __hash__(self) -> int: self.remote_type, self.push, self.url, + self.protocol, self.instance_name, self.server_cert_file, self.client_key_file, @@ -141,7 +158,7 @@ def __hash__(self) -> int: ) def __str__(self) -> str: - string = self.url + "\n" + string = f"{self.url} ({self.protocol})\n" string += "push: {} type: {} instance: {}\n".format(self.push, self.remote_type, self.instance_name) if self._spec_node: provenance = str(self._spec_node.get_provenance()) @@ -210,6 +227,7 @@ def request_timeout(self) -> Optional[int]: # def to_localcas_remote(self, remote): remote.url = self.url + remote.protocol = self.protocol if self.instance_name: remote.instance_name = self.instance_name if self.server_cert: @@ -264,7 +282,7 @@ def new_from_node( push: bool = False remote_type: str = RemoteType.ENDPOINT - valid_keys: List[str] = ["url", "instance-name", "auth", "connection-config"] + valid_keys: List[str] = ["url", "protocol", "instance-name", "auth", "connection-config"] if not remote_execution: remote_type = cast(str, spec_node.get_enum("type", RemoteType, default=RemoteType.ALL)) valid_keys += ["type"] @@ -283,6 +301,8 @@ def new_from_node( provenance = spec_node.get_node("url").get_provenance() raise LoadError("{}: empty artifact cache URL".format(provenance), LoadErrorReason.INVALID_DATA) + protocol = spec_node.get_str("protocol", default="grpc") + instance_name = spec_node.get_str("instance-name", default=None) auth_node = spec_node.get_mapping("auth", None) @@ -296,6 +316,7 @@ def new_from_node( return cls( remote_type, url, + protocol=protocol, push=push, server_cert=server_cert, client_key=client_key, @@ -327,6 +348,7 @@ def new_from_node( @classmethod def new_from_string(cls, string: str, purpose: int = RemoteSpecPurpose.ALL) -> "RemoteSpec": url: Optional[str] = None + protocol: Optional[str] = None instance_name: Optional[str] = None remote_type: str = RemoteType.ALL push: bool = True @@ -353,6 +375,8 @@ def new_from_string(cls, string: str, purpose: int = RemoteSpecPurpose.ALL) -> " if key == "url": url = val + elif key == "protocol": + protocol = val elif key == "instance-name": instance_name = val elif key == "type": @@ -396,6 +420,7 @@ def new_from_string(cls, string: str, purpose: int = RemoteSpecPurpose.ALL) -> " return cls( remote_type, url, + protocol=protocol, push=push, server_cert=server_cert, client_key=client_key, diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py index 83607c74f..e1f4afe68 100644 --- a/tests/artifactcache/pull.py +++ b/tests/artifactcache/pull.py @@ -16,6 +16,7 @@ # pylint: disable=redefined-outer-name import os +import shutil import pytest @@ -24,7 +25,7 @@ from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from buildstream._testing import cli # pylint: disable=unused-import -from tests.testutils import create_artifact_share, dummy_context +from tests.testutils import create_artifact_share, create_artifact_and_http_share, dummy_context # Project directory DATA_DIR = os.path.join( @@ -126,6 +127,94 @@ def test_pull(cli, tmpdir, datafiles, connection_config): assert cli.artifact.is_cached(cache_dir, element, element_key) +@pytest.mark.datafiles(DATA_DIR) +def test_pull_http(cli, tmpdir, datafiles): + project_dir = str(datafiles) + + # Set up an artifact cache. + with create_artifact_and_http_share(os.path.join(str(tmpdir), "artifactshare")) as (share, httpserver): + # Configure artifact share + cache_dir = os.path.join(str(tmpdir), "cache") + user_config_file = str(tmpdir.join("buildstream.conf")) + server = { + "url": share.repo, + "push": True, + } + user_config = { + "scheduler": {"pushers": 1}, + "artifacts": {"servers": [server]}, + "cachedir": cache_dir, + } + + # Write down the user configuration file + _yaml.roundtrip_dump(user_config, file=user_config_file) + # Ensure CLI calls will use it + cli.configure(user_config) + + # First build the project with the artifact cache configured + result = cli.run(project=project_dir, args=["build", "target.bst"]) + result.assert_success() + + # Assert that we are now cached locally + assert cli.get_element_state(project_dir, "target.bst") == "cached" + # Assert that we shared/pushed the cached artifact + assert share.get_artifact(cli.get_artifact_name(project_dir, "test", "target.bst")) + + # Now we've pushed, clear the user's local cache + cas = os.path.join(cli.directory, "cas") + shutil.rmtree(cas) + artifact_dir = os.path.join(cli.directory, "artifacts") + shutil.rmtree(artifact_dir) + + # Assert that we are not cached locally anymore + assert cli.get_element_state(project_dir, "target.bst") != "cached" + + # Switch gRPC server to index-only and add HTTP server + server["type"] = "index" + server2 = { + "url": httpserver.base_url(), + "type": "storage", + "protocol": "http", + } + user_config["artifacts"]["servers"] = [server, server2] + + # Write down the user configuration file + _yaml.roundtrip_dump(user_config, file=user_config_file) + # Ensure CLI calls will use it + cli.configure(user_config) + + with dummy_context(config=user_config_file) as context: + # Load the project + project = Project(project_dir, context) + project.ensure_fully_loaded() + + # Assert that the element's artifact is **not** cached + element = project.load_elements(["target.bst"])[0] + element_key = cli.get_element_key(project_dir, "target.bst") + assert not cli.artifact.is_cached(cache_dir, element, element_key) + + context.cachedir = cache_dir + context.casdir = os.path.join(cache_dir, "cas") + context.tmpdir = os.path.join(cache_dir, "tmp") + + # Load the project manually + project = Project(project_dir, context) + project.ensure_fully_loaded() + + # Create a local artifact cache handle + artifactcache = context.artifactcache + + # Initialize remotes + context.initialize_remotes(True, True, None, None) + + assert artifactcache.has_fetch_remotes( + plugin=element + ), "No remote configured for element target.bst (likely due to missing HTTP REST protocol support in buildbox-casd)" + assert artifactcache.pull(element, element_key), "Pull operation failed" + + assert cli.artifact.is_cached(cache_dir, element, element_key) + + @pytest.mark.datafiles(DATA_DIR) def test_pull_tree(cli, tmpdir, datafiles): project_dir = str(datafiles) diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py index 03eb422ea..4885fefc6 100644 --- a/tests/testutils/__init__.py +++ b/tests/testutils/__init__.py @@ -19,7 +19,14 @@ # William Salmon # -from .artifactshare import create_artifact_share, create_split_share, assert_shared, assert_not_shared, ArtifactShare +from .artifactshare import ( + create_artifact_share, + create_artifact_and_http_share, + create_split_share, + assert_shared, + assert_not_shared, + ArtifactShare, +) from .casd import casd_cache from .context import dummy_context from .element_generators import create_element_size diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py index fcdbcb597..66958eeb2 100644 --- a/tests/testutils/artifactshare.py +++ b/tests/testutils/artifactshare.py @@ -30,6 +30,8 @@ from buildstream._protos.buildstream.v2 import artifact_pb2 from buildstream._protos.google.rpc import code_pb2 +from .http_cas_server import HTTPCASServer + REMOTE_ASSET_ARTIFACT_URN_TEMPLATE = "urn:fdc:buildstream.build:2020:artifact:{}" REMOTE_ASSET_SOURCE_URN_TEMPLATE = "urn:fdc:buildstream.build:2020:source:{}" @@ -329,6 +331,22 @@ def create_split_share(directory1, directory2, *, quota=None): storage.close() +@contextmanager +def create_artifact_and_http_share(directory, *, quota=None): + share = ArtifactShare(directory, quota=quota) + try: + # Read-only access to the same CAS objects using a HTTP REST protocol + # https://github.com/buchgr/bazel-remote/ + httpserver = HTTPCASServer(share.repodir) + httpserver.start() + try: + yield share, httpserver + finally: + httpserver.stop() + finally: + share.close() + + # create_dummy_artifact_share() # # Create a dummy artifact share that doesn't have any capabilities diff --git a/tests/testutils/http_cas_server.py b/tests/testutils/http_cas_server.py new file mode 100644 index 000000000..7202d2f76 --- /dev/null +++ b/tests/testutils/http_cas_server.py @@ -0,0 +1,79 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import functools +import os +import re +import shutil +import threading +from http.server import BaseHTTPRequestHandler, HTTPStatus, ThreadingHTTPServer + + +class HTTPCASRequestHandler(BaseHTTPRequestHandler): + HASH_SIZE = 64 # SHA256 as hex digits + + def __init__(self, directory, *args, **kwargs): + self.objdir = os.path.join(directory, "cas", "objects") + super().__init__(*args, **kwargs) + + def do_GET(self): + self.process_get_or_head() + + def do_HEAD(self): + self.process_get_or_head() + + def process_get_or_head(self): + match = re.fullmatch(r"/cas/([0-9a-f]+)", self.path) + if match is None: + self.send_error(HTTPStatus.NOT_FOUND) + return + cas_hash = match.group(1) + if len(cas_hash) != self.HASH_SIZE: + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + with open(os.path.join(self.objdir, cas_hash[:2], cas_hash[2:]), "rb") as f: + fs = os.fstat(f.fileno()) + + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", str(fs.st_size)) + self.end_headers() + + if self.command == "GET": + shutil.copyfileobj(f, self.wfile) + except OSError: + self.send_error(HTTPStatus.NOT_FOUND) + + +class HTTPCASServer(threading.Thread): + def __init__(self, directory): + super().__init__() + self.server = ThreadingHTTPServer(("127.0.0.1", 0), functools.partial(HTTPCASRequestHandler, directory)) + self.started = False + + def start(self): + self.started = True + super().start() + + def run(self): + self.server.serve_forever() + + def stop(self): + if not self.started: + return + self.server.shutdown() + self.join() + + def base_url(self): + return "http://127.0.0.1:{}".format(self.server.server_port)