From 446042fbc4c58f55308d90dae1762e64bf13a474 Mon Sep 17 00:00:00 2001 From: Sourabh Jain <45626610+SumoSourabh@users.noreply.github.com> Date: Fri, 3 Apr 2026 14:33:36 +0530 Subject: [PATCH] CP-13110 Add support for virtualToPhysicalDefinition in DVP code (#630) * CP-13110 Add support for virtualToPhysicalDefinition in DVP code * Resolving comments from review. --- common/setup.py | 2 +- libs/setup.py | 2 +- platform/setup.py | 2 +- .../dlpx/virtualization/platform/_linked.py | 152 +++++++++++++++++- .../platform/_plugin_classes.py | 43 ++++- .../dlpx/virtualization/platform/_virtual.py | 83 +++++++++- .../dlpx/virtualization/platform/operation.py | 2 + .../fake_generated_definitions.py | 19 +++ .../python/dlpx/virtualization/test_plugin.py | 129 ++++++++++++++- .../_internal/commands/build.py | 5 + .../plugin_template/schema_template.json | 5 + .../_internal/plugin_importer.py | 4 + .../virtualization/_internal/settings.cfg | 2 +- .../validation_schemas/plugin_importer.yaml | 25 ++- .../validation_schemas/plugin_schema.json | 4 + .../_internal/commands/test_build.py | 13 ++ .../_internal/commands/test_initialize.py | 5 +- .../dlpx/virtualization/_internal/conftest.py | 32 +++- .../_internal/engine_version.cfg | 2 +- .../_internal/test_package_util.py | 2 +- 20 files changed, 517 insertions(+), 16 deletions(-) diff --git a/common/setup.py b/common/setup.py index 97ac96a9..7ecfe15a 100644 --- a/common/setup.py +++ b/common/setup.py @@ -4,7 +4,7 @@ PYTHON_SRC = 'src/main/python' install_requires = [ - "dvp-api == 1.9.0", + "dvp-api == 1.10.0.dev0", "six >= 1.17, < 1.18", ] diff --git a/libs/setup.py b/libs/setup.py index 7d39c5be..4f69b703 100644 --- a/libs/setup.py +++ b/libs/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.9.0", + "dvp-api == 1.10.0.dev0", "dvp-common == {}".format(version), "six >= 1.17, < 1.18", ] diff --git a/platform/setup.py b/platform/setup.py index a63f91e8..3377837b 100644 --- a/platform/setup.py +++ b/platform/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.9.0", + "dvp-api == 1.10.0.dev0", "dvp-common == {}".format(version), "six >= 1.17, < 1.18", ] diff --git a/platform/src/main/python/dlpx/virtualization/platform/_linked.py b/platform/src/main/python/dlpx/virtualization/platform/_linked.py index eda09add..dbc2a94c 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_linked.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_linked.py @@ -13,7 +13,7 @@ from dlpx.virtualization.common.exceptions import PluginRuntimeError from dlpx.virtualization.platform import (DirectSource, Mount, MountSpecification, StagedSource, - Status) + Status, PhysicalSource) from dlpx.virtualization.platform import validation_util as v from dlpx.virtualization.platform.exceptions import ( IncorrectReturnTypeError, OperationAlreadyDefinedError, @@ -33,6 +33,7 @@ def __init__(self): self.worker_impl = None self.mount_specification_impl = None self.source_size_impl = None + self.source_to_physical_impl = None def pre_snapshot(self): def pre_snapshot_decorator(pre_snapshot_impl): @@ -112,6 +113,16 @@ def source_size_decorator(source_size_impl): return source_size_decorator + def source_to_physical(self): + def source_to_physical_decorator(source_to_physical_impl): + if self.source_to_physical_impl: + raise OperationAlreadyDefinedError(Op.LINKED_SOURCE_TO_PHYSICAL) + self.source_to_physical_impl = v.check_function( + source_to_physical_impl, Op.LINKED_SOURCE_TO_PHYSICAL) + return source_to_physical_impl + + return source_to_physical_decorator + @staticmethod def _from_protobuf_remote_mount(remote_mount): return Mount(remote_environment=RemoteEnvironment.from_proto( @@ -323,6 +334,72 @@ def _internal_direct_source_size(self, request): return direct_source_size_response + def _internal_direct_source_to_physical(self, request): + """Direct Source to Physical Wrapper for direct plugins. + + Executed as part of several operations to convert a virtual source + to a physical source for a direct source. + + Run virtual_to_physical operation for a direct source. + + Args: + request (DirectSourceToPhysicalRequest): Virtual to Physical Request + arguments. + + Returns: + DirectSourceToPhysicalResponse: A response containing the return value - + DirectSourceToPhysicalResult. In case of errors, response object will + contain PluginErrorResult. + """ + # Reasoning for method imports are in this file's docstring. + from generated.definitions import RepositoryDefinition + from generated.definitions import LinkedSourceDefinition + from generated.definitions import SourceConfigDefinition + from generated.definitions import SnapshotDefinition + from generated.definitions import VirtualToPhysicalDefinition + + # + # While linked.virtual_to_physical() is not a required operation, + # this should not be called if it wasn't implemented. + # + if not self.source_to_physical_impl: + raise OperationNotDefinedError(Op.LINKED_SOURCE_TO_PHYSICAL) + + direct_source_definition = LinkedSourceDefinition.from_dict( + json.loads(request.direct_source.linked_source.parameters.json)) + direct_source = DirectSource( + guid=request.direct_source.linked_source.guid, + connection=RemoteConnection.from_proto( + request.direct_source.connection), + parameters=direct_source_definition) + virtual_to_physical_source_definition = VirtualToPhysicalDefinition.from_dict( + json.loads(request.physical_source.parameters.json)) + physical_source = PhysicalSource( + guid=request.physical_source.guid, + connection=RemoteConnection.from_proto(request.physical_source.connection), + target_directory=request.physical_source.target_directory, + parameters=virtual_to_physical_source_definition) + + repository = RepositoryDefinition.from_dict( + json.loads(request.repository.parameters.json)) + source_config = SourceConfigDefinition.from_dict( + json.loads(request.source_config.parameters.json)) + snapshot = SnapshotDefinition.from_dict( + json.loads(request.snapshot.parameters.json)) + + self.source_to_physical_impl( + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + direct_source_to_physical_response = ( + platform_pb2.DirectSourceToPhysicalResponse()) + direct_source_to_physical_response.return_value.CopyFrom( + platform_pb2.DirectSourceToPhysicalResult()) + return direct_source_to_physical_response + def _internal_staged_pre_snapshot(self, request): """Pre Snapshot Wrapper for staged plugins. @@ -852,3 +929,76 @@ def _internal_staged_source_size(self, request): staged_source_size_response.return_value.database_size = source_size return staged_source_size_response + + def _internal_staged_source_to_physical(self, request): + """Staged Source to Physical Wrapper for staged plugins. + + Executed as part of several operations to convert a virtual source + to a physical source for a staged source. + + Run virtual_to_physical operation for a staged source. + + Args: + request (StagedSourceToPhysicalRequest): Virtual to Physical Request + arguments. + + Returns: + StagedSourceToPhysicalResponse: A response containing the return value - + StagedSourceToPhysicalResult. In case of errors, response object will + contain PluginErrorResult. + """ + # Reasoning for method imports are in this file's docstring. + from generated.definitions import RepositoryDefinition + from generated.definitions import LinkedSourceDefinition + from generated.definitions import SourceConfigDefinition + from generated.definitions import SnapshotDefinition + from generated.definitions import VirtualToPhysicalDefinition + + # + # While linked.virtual_to_physical() is not a required operation, + # this should not be called if it wasn't implemented. + # + if not self.source_to_physical_impl: + raise OperationNotDefinedError(Op.LINKED_SOURCE_TO_PHYSICAL) + + staged_source_definition = LinkedSourceDefinition.from_dict( + json.loads(request.staged_source.linked_source.parameters.json)) + staged_mount, mounts = LinkedOperations._get_mounts_from_request(request) + staged_source = StagedSource( + guid=request.staged_source.linked_source.guid, + source_connection=RemoteConnection.from_proto( + request.staged_source.source_connection), + parameters=staged_source_definition, + mount=staged_mount, + staged_connection=RemoteConnection.from_proto( + request.staged_source.staged_connection), + mounts=mounts) + + virtual_to_physical_source_definition = VirtualToPhysicalDefinition.from_dict( + json.loads(request.physical_source.parameters.json)) + physical_source = PhysicalSource( + guid=request.physical_source.guid, + connection=RemoteConnection.from_proto(request.physical_source.connection), + target_directory=request.physical_source.target_directory, + parameters=virtual_to_physical_source_definition) + + repository = RepositoryDefinition.from_dict( + json.loads(request.repository.parameters.json)) + source_config = SourceConfigDefinition.from_dict( + json.loads(request.source_config.parameters.json)) + snapshot = SnapshotDefinition.from_dict( + json.loads(request.snapshot.parameters.json)) + + self.source_to_physical_impl( + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source + ) + + staged_source_to_physical_response = ( + platform_pb2.StagedSourceToPhysicalResponse()) + staged_source_to_physical_response.return_value.CopyFrom( + platform_pb2.StagedSourceToPhysicalResult()) + return staged_source_to_physical_response diff --git a/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py b/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py index bc9cf47a..e26f8046 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py @@ -20,8 +20,9 @@ of the autogenerated classes from the schemas (e.g. VirtualSourceDefinition) """ __all__ = [ - "VirtualSource", "StagedSource", "DirectSource", "Status", "MigrationType", - "Mount", "OwnershipSpecification", "MountSpecification" + "VirtualSource", "StagedSource", "DirectSource", "PhysicalSource", + "Status", "MigrationType", "Mount", "OwnershipSpecification", + "MountSpecification" ] @@ -137,6 +138,44 @@ def parameters(self): return self._parameters +class PhysicalSource(object): + def __init__(self, guid, connection, target_directory, parameters): + self._guid = guid + if not isinstance(connection, RemoteConnection): + raise IncorrectTypeError(RemoteConnection, 'connection', + type(connection), RemoteConnection) + self._connection = connection + if not isinstance(target_directory, six.string_types): + raise IncorrectTypeError(PhysicalSource, 'target_directory', + type(target_directory), + six.string_types[0]) + self._target_directory = target_directory + self._parameters = parameters + + @property + def guid(self): + """str: The unique guid identifier for this PhysicalSource.""" + return self._guid + + @property + def connection(self): + """RemoteConnection: The RemoteConnection for this + PhysicalSource.""" + return self._connection + + @property + def target_directory(self): + """str: The target directory path for this PhysicalSource.""" + return self._target_directory + + @property + def parameters(self): + """PhysicalSourceDefinition: The PhysicalSourceDefinition for this + PhysicalSource. + """ + return self._parameters + + class Status(Enum): ACTIVE = 0 INACTIVE = 1 diff --git a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py index 05fa5cd2..f947efad 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py @@ -11,7 +11,7 @@ from dlpx.virtualization.api import common_pb2, platform_pb2 from dlpx.virtualization.common import RemoteConnection, RemoteEnvironment from dlpx.virtualization.platform import (Mount, MountSpecification, Status, - VirtualSource) + VirtualSource, PhysicalSource) from dlpx.virtualization.platform import validation_util as v from dlpx.virtualization.platform.exceptions import ( IncorrectReturnTypeError, OperationAlreadyDefinedError, @@ -35,6 +35,7 @@ def __init__(self): self.initialize_impl = None self.mount_specification_impl = None self.source_size_impl = None + self.source_to_physical_impl = None def configure(self): def configure_decorator(configure_impl): @@ -153,6 +154,17 @@ def source_size_decorator(source_size_impl): return source_size_decorator + def source_to_physical(self): + def source_to_physical_decorator(source_to_physical_impl): + if self.source_to_physical_impl: + raise OperationAlreadyDefinedError( + Op.VIRTUAL_SOURCE_TO_PHYSICAL) + self.source_to_physical_impl = v.check_function( + source_to_physical_impl, Op.VIRTUAL_SOURCE_TO_PHYSICAL) + return source_to_physical_impl + + return source_to_physical_decorator + @staticmethod def _from_protobuf_single_subset_mount(single_subset_mount): return Mount(remote_environment=RemoteEnvironment.from_proto( @@ -848,3 +860,72 @@ def _internal_virtual_source_size(self, request): virtual_source_size_response.return_value.database_size = source_size return virtual_source_size_response + + def _internal_virtual_source_to_physical(self, request): + """Virtual to Physical Wrapper. + + Executed as part of several operations to convert a virtual source + to a physical source. + + Run virtual_to_physical operation for a virtual source. + + Args: + request (VirtualSourceToPhysicalRequest): Virtual to Physical Request + arguments. + + Returns: + VirtualSourceToPhysicalResponse: A response containing the return value - + VirtualSourceToPhysicalResult. In case of errors, response object will + contain PluginErrorResult. + """ + # Reasoning for method imports are in this file's docstring. + from generated.definitions import VirtualSourceDefinition + from generated.definitions import RepositoryDefinition + from generated.definitions import SourceConfigDefinition + from generated.definitions import SnapshotDefinition + from generated.definitions import VirtualToPhysicalDefinition + + # + # While virtual.virtual_to_physical() is not a required operation, + # this should not be called if it wasn't implemented. + # + if not self.source_to_physical_impl: + raise OperationNotDefinedError(Op.VIRTUAL_SOURCE_TO_PHYSICAL) + + virtual_source_definition = VirtualSourceDefinition.from_dict( + json.loads(request.virtual_source.parameters.json)) + mounts = [ + VirtualOperations._from_protobuf_single_subset_mount(m) + for m in request.virtual_source.mounts + ] + virtual_source = VirtualSource(guid=request.virtual_source.guid, + connection=RemoteConnection.from_proto( + request.virtual_source.connection), + parameters=virtual_source_definition, + mounts=mounts) + virtual_to_physical_source_definition = VirtualToPhysicalDefinition.from_dict( + json.loads(request.physical_source.parameters.json)) + physical_source = PhysicalSource( + guid=request.physical_source.guid, + connection=RemoteConnection.from_proto(request.physical_source.connection), + target_directory=request.physical_source.target_directory, + parameters=virtual_to_physical_source_definition) + repository = RepositoryDefinition.from_dict( + json.loads(request.repository.parameters.json)) + source_config = SourceConfigDefinition.from_dict( + json.loads(request.source_config.parameters.json)) + snapshot = SnapshotDefinition.from_dict( + json.loads(request.snapshot.parameters.json)) + + self.source_to_physical_impl( + virtual_source=virtual_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + virtual_to_physical_response = ( + platform_pb2.VirtualSourceToPhysicalResponse()) + virtual_to_physical_response.return_value.CopyFrom( + platform_pb2.VirtualSourceToPhysicalResult()) + return virtual_to_physical_response diff --git a/platform/src/main/python/dlpx/virtualization/platform/operation.py b/platform/src/main/python/dlpx/virtualization/platform/operation.py index 6d033d21..8eeeef5f 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/operation.py +++ b/platform/src/main/python/dlpx/virtualization/platform/operation.py @@ -18,6 +18,7 @@ class Operation(Enum): LINKED_WORKER = 'linked.worker()' LINKED_MOUNT_SPEC = 'linked.mount_specification()' LINKED_SOURCE_SIZE = 'linked.source_size()' + LINKED_SOURCE_TO_PHYSICAL = 'linked.source_to_physical()' VIRTUAL_CONFIGURE = 'virtual.configure()' VIRTUAL_UNCONFIGURE = 'virtual.unconfigure()' @@ -31,6 +32,7 @@ class Operation(Enum): VIRTUAL_INITIALIZE = 'virtual.initialize()' VIRTUAL_MOUNT_SPEC = 'virtual.mount_specification()' VIRTUAL_SOURCE_SIZE = 'virtual.source_size()' + VIRTUAL_SOURCE_TO_PHYSICAL = 'virtual.source_to_physical()' UPGRADE_REPOSITORY = 'upgrade.repository()' UPGRADE_SOURCE_CONFIG = 'upgrade.source_config()' diff --git a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py index e4ef3419..9b5d087c 100644 --- a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py +++ b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py @@ -120,3 +120,22 @@ def from_dict(input_dict): def to_dict(self): return {"resync": self._resync} + + +class VirtualToPhysicalDefinition(Model): + def __init__(self, name): + self.swagger_types = {'name': six.string_types[0]} + + self.attribute_map = {'name': 'name'} + self._name = name + + @property + def name(self): + return self._name + + @staticmethod + def from_dict(input_dict): + return VirtualToPhysicalDefinition(input_dict['name']) + + def to_dict(self): + return {"name": self._name} diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin.py b/platform/src/test/python/dlpx/virtualization/test_plugin.py index f687549e..fb7adbf5 100755 --- a/platform/src/test/python/dlpx/virtualization/test_plugin.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin.py @@ -39,6 +39,7 @@ TEST_DIRECT_SOURCE = 'TestDirectSource' TEST_STAGED_SOURCE = 'TestStagedSource' TEST_VIRTUAL_SOURCE = 'TestVirtualSource' +TEST_PHYSICAL_SOURCE = 'TestPhysicalSource' # This is a simple JSON object that has only "name" property defined. SIMPLE_JSON = '{{"name": "{0}"}}' @@ -49,6 +50,7 @@ TEST_DIRECT_SOURCE_JSON = SIMPLE_JSON.format(TEST_DIRECT_SOURCE) TEST_STAGED_SOURCE_JSON = SIMPLE_JSON.format(TEST_STAGED_SOURCE) TEST_VIRTUAL_SOURCE_JSON = SIMPLE_JSON.format(TEST_VIRTUAL_SOURCE) +TEST_PHYSICAL_SOURCE_JSON = SIMPLE_JSON.format(TEST_PHYSICAL_SOURCE) TEST_SNAPSHOT_PARAMS_JSON = '{"resync": false}' TEST_PRE_UPGRADE_PARAMS = {'obj': json.dumps({'name': 'upgrade'})} TEST_POST_MIGRATION_METADATA_1 = (json.dumps( @@ -157,6 +159,10 @@ def assert_mounts(mounts): assert len(mounts) == 1 TestPlugin.assert_mount(mounts[0]) + @staticmethod + def assert_target_directory(target_directory): + assert target_directory == TEST_BINARY_PATH + @staticmethod def assert_virtual_source(virtual_source): assert virtual_source.guid == TEST_GUID @@ -204,6 +210,13 @@ def assert_snapshot(snapshot): def assert_snapshot_parameters(snapshot_parameters): assert not snapshot_parameters.resync + @staticmethod + def assert_physical_source(physical_source): + assert physical_source.guid == TEST_GUID + TestPlugin.assert_connection(physical_source.connection) + TestPlugin.assert_target_directory(physical_source.target_directory) + assert physical_source.parameters.name == TEST_PHYSICAL_SOURCE + @staticmethod @pytest.fixture def host(): @@ -232,6 +245,11 @@ def mount(environment): mount.shared_path = TEST_SHARED_PATH return mount + @staticmethod + @pytest.fixture + def target_directory(): + return TEST_BINARY_PATH + @staticmethod @pytest.fixture def user(): @@ -266,6 +284,16 @@ def virtual_source(connection, mount): virtual_source.mounts.extend([mount]) return virtual_source + @staticmethod + @pytest.fixture + def physical_source(connection, target_directory): + physical_source = common_pb2.PhysicalSource() + physical_source.guid = TEST_GUID + physical_source.connection.CopyFrom(connection) + physical_source.parameters.json = TEST_PHYSICAL_SOURCE_JSON + physical_source.target_directory = target_directory + return physical_source + @staticmethod @pytest.fixture def repository(): @@ -342,7 +370,8 @@ def setup_request(request, repository=None, source_config=None, snapshot=None, - snapshot_parameters=None): + snapshot_parameters=None, + physical_source=None): if virtual_source: request.virtual_source.CopyFrom(virtual_source) @@ -364,6 +393,9 @@ def setup_request(request, if snapshot_parameters: request.snapshot_parameters.CopyFrom(snapshot_parameters) + if physical_source: + request.physical_source.CopyFrom(physical_source) + @staticmethod def assert_plugin_args(**kwargs): for key, value in kwargs.items(): @@ -391,6 +423,9 @@ def assert_plugin_args(**kwargs): TestPlugin.assert_snapshot_parameters( kwargs['snapshot_parameters']) + if 'physical_source' in kwargs: + TestPlugin.assert_physical_source(kwargs['physical_source']) + @staticmethod def test_virtual_configure(my_plugin, virtual_source, repository, snapshot): @@ -792,6 +827,35 @@ def virtual_source_size_impl(virtual_source, repository, source_config): assert virtual_source_size_response.return_value.database_size == 100 + @staticmethod + def test_virtual_source_to_physical(my_plugin, virtual_source, repository, + source_config, snapshot, physical_source): + + @my_plugin.virtual.source_to_physical() + def virtual_source_to_physical_impl(virtual_source, repository, + source_config, snapshot, physical_source): + TestPlugin.assert_plugin_args(virtual_source=virtual_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + virtual_to_physical_request = platform_pb2.VirtualSourceToPhysicalRequest() + TestPlugin.setup_request(request=virtual_to_physical_request, + virtual_source=virtual_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + virtual_to_physical_response = my_plugin.virtual. \ + _internal_virtual_source_to_physical(virtual_to_physical_request) + expected_result = platform_pb2.VirtualSourceToPhysicalResult() + # Check that the response's oneof is set to return_value and not error + assert virtual_to_physical_response.WhichOneof( + 'result') == 'return_value' + assert virtual_to_physical_response.return_value == expected_result + @staticmethod def test_repository_discovery(my_plugin, connection): @my_plugin.discovery.repository() @@ -1009,6 +1073,35 @@ def direct_source_size_impl(direct_source, repository, source_config): assert direct_source_size_response.return_value.database_size == 100 + @staticmethod + def test_direct_source_to_physical(my_plugin, direct_source, repository, + source_config, snapshot, physical_source): + + @my_plugin.linked.source_to_physical() + def direct_source_to_physical_impl(direct_source, repository, + source_config, snapshot, physical_source): + TestPlugin.assert_plugin_args(direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + direct_to_physical_request = platform_pb2.DirectSourceToPhysicalRequest() + TestPlugin.setup_request(request=direct_to_physical_request, + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + direct_to_physical_response = my_plugin.linked. \ + _internal_direct_source_to_physical(direct_to_physical_request) + expected_result = platform_pb2.DirectSourceToPhysicalResult() + # Check that the response's oneof is set to return_value and not error + assert direct_to_physical_response.WhichOneof( + 'result') == 'return_value' + assert direct_to_physical_response.return_value == expected_result + @staticmethod @pytest.mark.parametrize("staged_source", ["mount", "mounts", "Both"], indirect=["staged_source"]) @@ -1249,6 +1342,40 @@ def staged_source_size_impl(staged_source, repository, source_config): if staged_source_size_response: assert staged_source_size_response.return_value.database_size == 0 + @staticmethod + @pytest.mark.parametrize("staged_source", ["mount", "mounts", "Both"], + indirect=["staged_source"]) + def test_staged_source_to_physical(my_plugin, staged_source, repository, + source_config, snapshot, physical_source): + + @my_plugin.linked.source_to_physical() + def staged_source_to_physical_impl(staged_source, repository, + source_config, snapshot, physical_source): + TestPlugin.assert_plugin_args(staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + staged_to_physical_request = platform_pb2.StagedSourceToPhysicalRequest() + TestPlugin.setup_request(request=staged_to_physical_request, + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + staged_to_physical_response = TestPlugin._call_stage_methods( + my_plugin.linked._internal_staged_source_to_physical, + staged_to_physical_request, + TestPlugin._raise_staged_source_both_mounts_exception(staged_source)) + expected_result = platform_pb2.StagedSourceToPhysicalResult() + # Check that the response's oneof is set to return_value and not error + if staged_to_physical_response: + assert staged_to_physical_response.WhichOneof( + 'result') == 'return_value' + assert staged_to_physical_response.return_value == expected_result + @staticmethod @pytest.mark.parametrize("staged_source", ["mount", "mounts", "Both"], indirect=["staged_source"]) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py index 7da2e8bd..68391641 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py @@ -21,6 +21,7 @@ TYPE = 'Plugin' LOCALE_DEFAULT = 'en-us' VIRTUAL_SOURCE_TYPE = 'PluginVirtualSourceDefinition' +VIRTUAL_TO_PHYSICAL_TYPE = 'PluginVirtualToPhysicalDefinition' DISCOVERY_DEFINITION_TYPE = 'PluginDiscoveryDefinition' STAGED_LINKED_SOURCE_TYPE = 'PluginLinkedStagedSourceDefinition' DIRECT_LINKED_SOURCE_TYPE = 'PluginLinkedDirectSourceDefinition' @@ -243,6 +244,10 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): 'type': VIRTUAL_SOURCE_TYPE, 'parameters': schemas['virtualSourceDefinition'] }, + 'virtualToPhysicalDefinition': { + 'type': VIRTUAL_TO_PHYSICAL_TYPE, + 'parameters': schemas['virtualToPhysicalDefinition'] + }, 'linkedSourceDefinition': { 'type': get_linked_source_definition_type(plugin_config_content), 'parameters': schemas['linkedSourceDefinition'] diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json index 8dd02073..7f629d50 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json @@ -22,6 +22,11 @@ "additionalProperties" : false, "properties" : {} }, + "virtualToPhysicalDefinition": { + "type": "object", + "additionalProperties" : false, + "properties" : {} + }, "linkedSourceDefinition": { "type": "object", "additionalProperties" : false, diff --git a/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py b/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py index f569c120..48ff97a4 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py @@ -360,6 +360,8 @@ def _prepare_manifest(entry_point, module_content): bool(plugin_object.linked.mount_specification_impl), 'hasLinkedSourceSize': bool(plugin_object.linked.source_size_impl), + 'hasLinkedSourceToPhysical': + bool(plugin_object.linked.source_to_physical_impl), 'hasVirtualConfigure': bool(plugin_object.virtual.configure_impl), 'hasVirtualUnconfigure': @@ -384,6 +386,8 @@ def _prepare_manifest(entry_point, module_content): bool(plugin_object.virtual.initialize_impl), 'hasVirtualSourceSize': bool(plugin_object.virtual.source_size_impl), + 'hasVirtualSourceToPhysical': + bool(plugin_object.virtual.source_to_physical_impl), 'migrationIdList': plugin_object.upgrade.migration_id_list } diff --git a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg index c80f2be1..21d61d2a 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg +++ b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg @@ -20,7 +20,7 @@ # versions in those packages until they are shipped out of band. # [General] -engine_api_version = 1.11.16 +engine_api_version = 1.11.49 distribution_name = dvp-tools package_author = Delphix namespace_package = dlpx diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml index 9193e581..018da1e8 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml @@ -42,6 +42,12 @@ EXPECTED_STAGED_ARGS_BY_OP: - staged_source - repository - source_config + source_to_physical_impl: + - staged_source + - repository + - source_config + - snapshot + - physical_source VirtualOperations: configure_impl: - virtual_source @@ -90,7 +96,12 @@ EXPECTED_STAGED_ARGS_BY_OP: - virtual_source - repository - source_config - + source_to_physical_impl: + - virtual_source + - repository + - source_config + - snapshot + - physical_source EXPECTED_DIRECT_ARGS_BY_OP: @@ -115,6 +126,12 @@ EXPECTED_DIRECT_ARGS_BY_OP: - direct_source - repository - source_config + source_to_physical_impl: + - direct_source + - repository + - source_config + - snapshot + - physical_source VirtualOperations: configure_impl: - virtual_source @@ -163,6 +180,12 @@ EXPECTED_DIRECT_ARGS_BY_OP: - virtual_source - repository - source_config + source_to_physical_impl: + - virtual_source + - repository + - source_config + - snapshot + - physical_source EXPECTED_UPGRADE_ARGS: _repository_id_to_impl: diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json index 59c39057..ea1141dd 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json @@ -183,6 +183,7 @@ "repositoryDefinition", "sourceConfigDefinition", "virtualSourceDefinition", + "virtualToPhysicalDefinition", "linkedSourceDefinition", "snapshotDefinition", "snapshotParametersDefinition" @@ -218,6 +219,9 @@ } ] }, + "virtualToPhysicalDefinition": { + "$ref": "#/definitions/jsonSchema" + }, "linkedSourceDefinition": { "allOf": [ { diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py index b26be24e..c2cfc504 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py @@ -679,6 +679,19 @@ def test_plugin_missing_schema_def(mock_generate_python, assert not mock_generate_python.called + @staticmethod + @pytest.mark.parametrize('virtual_to_physical_definition', [None]) + @mock.patch('dlpx.virtualization._internal.codegen.generate_python') + def test_plugin_missing_virtual_to_physical_schema_def( + mock_generate_python, plugin_config_file, artifact_file): + with pytest.raises(exceptions.UserError) as err_info: + build.build(plugin_config_file, artifact_file, False, False) + + message = err_info.value.message + assert "'virtualToPhysicalDefinition' is a required property" in message + + assert not mock_generate_python.called + @staticmethod @mock.patch('dlpx.virtualization._internal.codegen.generate_python') @pytest.mark.parametrize('additional_definition', [{ diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py index ecdfa4e5..67cee89a 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py @@ -270,11 +270,12 @@ def test_default_schema_definition(schema_template): ] # - # Validate the linked source, virtual source, and snapshot definitions - # have no properties. + # Validate the linked source, virtual source, virtual to physical, and + # snapshot definitions have no properties. # assert schema_template['linkedSourceDefinition']['properties'] == {} assert schema_template['virtualSourceDefinition']['properties'] == {} + assert schema_template['virtualToPhysicalDefinition']['properties'] == {} assert schema_template['snapshotDefinition']['properties'] == {} @staticmethod diff --git a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py index f940f7cc..e9db3e2d 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py @@ -371,6 +371,7 @@ def post_snapshot(direct_source, repository, source_config): linked.worker_impl = None linked.mount_specification_impl = None linked.source_size_impl = None + linked.source_to_physical_impl = None return linked @@ -412,6 +413,7 @@ def mount_specification(virtual_source, repository): virtual.initialize_impl = None virtual.cleanup_impl = None virtual.source_size_impl = None + virtual.source_to_physical_impl = None return virtual @@ -446,6 +448,7 @@ def plugin_manifest(upgrade_operation): 'hasLinkedWorker': False, 'hasLinkedMountSpecification': False, 'hasLinkedSourceSize': False, + 'hasLinkedSourceToPhysical': False, 'hasVirtualConfigure': True, 'hasVirtualUnconfigure': False, 'hasVirtualReconfigure': True, @@ -459,13 +462,15 @@ def plugin_manifest(upgrade_operation): 'hasInitialize': False, 'migrationIdList': upgrade_operation.migration_id_list, 'hasVirtualCleanup': False, + 'hasVirtualSourceToPhysical': False, } return manifest @pytest.fixture def schema_content(repository_definition, source_config_definition, - virtual_source_definition, linked_source_definition, + virtual_source_definition, virtual_to_physical_definition, + linked_source_definition, snapshot_definition, snapshot_parameters_definition, additional_definition): @@ -480,6 +485,9 @@ def schema_content(repository_definition, source_config_definition, if virtual_source_definition: schema['virtualSourceDefinition'] = virtual_source_definition + if virtual_to_physical_definition: + schema['virtualToPhysicalDefinition'] = virtual_to_physical_definition + if linked_source_definition: schema['linkedSourceDefinition'] = linked_source_definition @@ -541,6 +549,19 @@ def virtual_source_definition(): } +@pytest.fixture +def virtual_to_physical_definition(): + return { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'path': { + 'type': 'string' + } + } + } + + @pytest.fixture def linked_source_definition(): return {'type': 'object', 'additionalProperties': False, 'properties': {}} @@ -649,7 +670,8 @@ def add_symlink_folder_to_src_dir(tmpdir, src_dir): @pytest.fixture def artifact_content(engine_api, virtual_source_definition, - linked_source_definition, discovery_definition, + virtual_to_physical_definition, linked_source_definition, + discovery_definition, snapshot_definition, snapshot_parameters_definition): """ This fixture creates base artifact that was generated from build and @@ -685,6 +707,12 @@ def artifact_content(engine_api, virtual_source_definition, 'parameters': virtual_source_definition, } + if virtual_to_physical_definition: + artifact['virtualToPhysicalDefinition'] = { + 'type': 'PluginVirtualToPhysicalDefinition', + 'parameters': virtual_to_physical_definition, + } + if linked_source_definition: artifact['linkedSourceDefinition'] = { 'type': 'PluginLinkedDirectSourceDefinition', diff --git a/tools/src/test/python/dlpx/virtualization/_internal/engine_version.cfg b/tools/src/test/python/dlpx/virtualization/_internal/engine_version.cfg index d5cb24b4..a3b69e2e 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/engine_version.cfg +++ b/tools/src/test/python/dlpx/virtualization/_internal/engine_version.cfg @@ -11,4 +11,4 @@ # This file also should not be updated manually with feature changes. # [General] -engine_api_version = 1.11.16 +engine_api_version = 1.11.49 diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index ce008ac5..9fe28878 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -8,7 +8,7 @@ from dlpx.virtualization._internal import package_util DVP_VERSION = '5.1.0' -DVP_API_VERSION = '1.9.0' +DVP_API_VERSION = '1.10.0' class TestPackageUtil: