diff --git a/common/setup.py b/common/setup.py index 97ac96a9..7ecfe15a 100644 --- a/common/setup.py +++ b/common/setup.py @@ -4,7 +4,7 @@ PYTHON_SRC = 'src/main/python' install_requires = [ - "dvp-api == 1.9.0", + "dvp-api == 1.10.0.dev0", "six >= 1.17, < 1.18", ] diff --git a/docs/docs/References/Classes.md b/docs/docs/References/Classes.md index 5e0b752d..573d0579 100644 --- a/docs/docs/References/Classes.md +++ b/docs/docs/References/Classes.md @@ -44,6 +44,25 @@ mounts | list[[Mount](#mount)] | Mount points associated with the source. > - When only one [mountSpecification](#mountspecification) object is provided, `mount` will be present. - When more than one [mountSpecification](#mountspecification) object is provided, `mounts` will be present. +## PhysicalSource + +Represents the target physical source when converting a virtual or linked source to a physical source. Passed as an argument to the [Virtual Source to Physical](Plugin_Operations.md#virtual-source-to-physical), [Direct Linked Source to Physical](Plugin_Operations.md#direct-linked-source-to-physical), and [Staged Linked Source to Physical](Plugin_Operations.md#staged-linked-source-to-physical) operations. + +```python +from dlpx.virtualization.platform import PhysicalSource + +physical_source = PhysicalSource(guid, connection, target_directory, parameters) +``` + +### Fields + +Field | Type | Description +----- | ---- | ----------- +guid | String | Unique identifier for the physical source. +connection | [RemoteConnection](#remoteconnection) | Connection for the target physical environment. +target_directory | String | The directory path on the physical host where data will be exported. +parameters | [VirtualToPhysicalDefinition](Schemas_and_Autogenerated_Classes.md#virtualtophysicaldefinition-class) | User input as per the [VirtualToPhysicalDefinition Schema](Schemas_and_Autogenerated_Classes.md#virtualtophysicaldefinition-schema). + ## VirtualSource Represents a Virtual Source object and its properties. diff --git a/docs/docs/References/Decorators.md b/docs/docs/References/Decorators.md index f48ac2e9..b5ba747a 100644 --- a/docs/docs/References/Decorators.md +++ b/docs/docs/References/Decorators.md @@ -28,6 +28,7 @@ Plugin Operation | Decorator [Direct Linked Source Pre-Snapshot](Plugin_Operations.md#direct-linked-source-pre-snapshot) | `@plugin.linked.pre_snapshot()` [Direct Linked Source Post-Snapshot](Plugin_Operations.md#direct-linked-source-post-snapshot) | `@plugin.linked.post_snapshot()` [Direct Linked Source Size](Plugin_Operations.md#direct-linked-source-size) | `@plugin.linked.source_size()` +[Direct Linked Source to Physical](Plugin_Operations.md#direct-linked-source-to-physical) | `@plugin.linked.source_to_physical()` [Staged Linked Source Pre-Snapshot](Plugin_Operations.md#staged-linked-source-pre-snapshot) | `@plugin.linked.pre_snapshot()` [Staged Linked Source Post-Snapshot](Plugin_Operations.md#staged-linked-source-post-snapshot) | `@plugin.linked.post_snapshot()` [Staged Linked Source Start-Staging](Plugin_Operations.md#staged-linked-source-start-staging) | `@plugin.linked.start_staging()` @@ -36,6 +37,7 @@ Plugin Operation | Decorator [Staged Linked Source Worker](Plugin_Operations.md#staged-linked-source-worker) | `@plugin.linked.worker()` [Staged Linked Source Mount Specification](Plugin_Operations.md#staged-linked-source-mount-specification) | `@plugin.linked.mount_specification()` [Staged Linked Source Size](Plugin_Operations.md#staged-linked-source-size) | `@plugin.linked.source_size()` +[Staged Linked Source to Physical](Plugin_Operations.md#staged-linked-source-to-physical) | `@plugin.linked.source_to_physical()` [Virtual Source Configure](Plugin_Operations.md#virtual-source-configure) | `@plugin.virtual.configure()` [Virtual Source Initialize](Plugin_Operations.md#virtual-source-initialize) | `@plugin.virtual.initialize()` [Virtual Source Unconfigure](Plugin_Operations.md#virtual-source-unconfigure) | `@plugin.virtual.unconfigure()` @@ -48,6 +50,7 @@ Plugin Operation | Decorator [Virtual Source Mount Specification](Plugin_Operations.md#virtual-source-mount-specification) | `@plugin.virtual.mount_specification()` [Virtual Source Status](Plugin_Operations.md#virtual-source-status) | `@plugin.virtual.status()` [Virtual Source Size](Plugin_Operations.md#virtual-source-size) | `@plugin.virtual.source_size()` +[Virtual Source to Physical](Plugin_Operations.md#virtual-source-to-physical) | `@plugin.virtual.source_to_physical()` [Repository Data Migration](Plugin_Operations.md#repository-data-migration) | `@plugin.upgrade.repository(migration_id)` [Source Config Data Migration](Plugin_Operations.md#source-config-data-migration) | `@plugin.upgrade.source_config(migration_id)` [Linked Source Data Migration](Plugin_Operations.md#linked-source-data-migration) | `@plugin.upgrade.linked_source(migration_id)` diff --git a/docs/docs/References/Glossary.md b/docs/docs/References/Glossary.md index 7aba6042..a1b949c6 100644 --- a/docs/docs/References/Glossary.md +++ b/docs/docs/References/Glossary.md @@ -65,6 +65,12 @@ A collection of information, provided by the plugin, which give all the details ## Password Properties In [schemas](#schema), any string property can be tagged with `"format": "password"`. This will let the Delphix Engine know that the property contains sensitive information. Any such values will only be stored in encrypted format, and the UI will not display the values on screen. +## Physical Dataset +A real, non-virtualized, physical copy of data that has been exported from the Delphix Engine to a target environment. Unlike a [virtual dataset](#virtual-dataset), a physical dataset is stored directly on the target host outside Delphix storage. + +## Physical Source +An object on the Delphix Engine that holds information related to a [physical dataset](#physical-dataset). + ## Platform Libraries A set of Python functions that are provided by the Virtualization Platform. Plugins use these library functions to request that the Virtualization Platform do some task on behalf of the plugin. For example, running a Bash command on an environment, or making an log entry. diff --git a/docs/docs/References/Plugin_Operations.md b/docs/docs/References/Plugin_Operations.md index fe0b108b..816f8c19 100644 --- a/docs/docs/References/Plugin_Operations.md +++ b/docs/docs/References/Plugin_Operations.md @@ -15,6 +15,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations [Direct Linked Source
Pre-Snapshot](#direct-linked-source-pre-snapshot) | **No** | `linked.pre_snapshot()` | [Linked Source Sync](Workflows.md#linked-source-sync) [Direct Linked Source
Post-Snapshot](#direct-linked-source-post-snapshot) | **Yes** | `linked.post_snapshot()` | [Linked Source Sync](Workflows.md#linked-source-sync) [Direct Linked Source
Source Size](#direct-linked-source-size) | **No** | `linked.source_size()` | N/A +[Direct Linked Source
to Physical](#direct-linked-source-to-physical) | **No** | `linked.source_to_physical()` | [Linked Source to Physical](Workflows.md#linked-source-to-physical) [Staged Linked Source
Pre-Snapshot](#staged-linked-source-pre-snapshot) | **No** | `linked.pre_snapshot()` | [Linked Source Sync](Workflows.md#linked-source-sync) [Staged Linked Source
Post-Snapshot](#staged-linked-source-post-snapshot) | **Yes** | `linked.post_snapshot()` | [Linked Source Sync](Workflows.md#linked-source-sync) [Staged Linked Source
Start-Staging](#staged-linked-source-start-staging) | **No** | `linked.start_staging()` | [Linked Source Enable](Workflows.md#linked-source-enable) @@ -23,6 +24,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations [Staged Linked Source
Worker](#staged-linked-source-worker) | **No** |`linked.worker()` | N/A [Staged Linked Source
Mount Specification](#staged-linked-source-mount-specification) | **Yes** | `linked.mount_specification()` | [Linked Source Sync](Workflows.md#linked-source-sync)
[Linked Source Enable](Workflows.md#linked-source-enable) [Staged Linked Source
Source Size](#staged-linked-source-size) | **No** | `linked.source_size()` | N/A +[Staged Linked Source
to Physical](#staged-linked-source-to-physical) | **No** | `linked.source_to_physical()` | [Linked Source to Physical](Workflows.md#linked-source-to-physical) [Virtual Source
Initialize](#virtual-source-initialize) | **No** | `virtual.initialize()` | [Virtual Source Create Empty VDB](Workflows.md#virtual-source-create-empty-vdb) [Virtual Source
Configure](#virtual-source-configure) | **Yes** | `virtual.configure()` | [Virtual Source Provision](Workflows.md#virtual-source-provision)
[Virtual Source Refresh](Workflows.md#virtual-source-refresh) [Virtual Source
Unconfigure](#virtual-source-unconfigure) | **No** | `virtual.unconfigure()` | [Virtual Source Refresh](Workflows.md#virtual-source-refresh)
[Virtual Source Delete](Workflows.md#virtual-source-delete) @@ -35,6 +37,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations [Virtual Source
Mount Specification](#virtual-source-mount-specification) | **Yes** | `virtual.mount_specification()` | [Virtual Source Enable](Workflows.md#virtual-source-enable)
[Virtual Source Provision](Workflows.md#virtual-source-provision)
[Virtual Source Refresh](Workflows.md#virtual-source-refresh)
[Virtual Source Rollback](Workflows.md#virtual-source-rollback)
[Virtual Source Start](Workflows.md#virtual-source-start) [Virtual Source
Status](#virtual-source-status) | **No** | `virtual.status()` | [Virtual Source Enable](Workflows.md#virtual-source-enable) [Virtual Source
Source Size](#virtual-source-size) | **No** | `virtual.source_size()` | N/A +[Virtual Source
to Physical](#virtual-source-to-physical) | **No** | `virtual.source_to_physical()` | [Virtual Source to Physical](Workflows.md#virtual-source-to-physical) [Repository Data Migration](#repository-data-migration) | **No** | `upgrade.repository(migration_id)` | [Upgrade](Workflows.md#upgrade) [Source Config Data Migration](#source-config-data-migration) | **No** | `upgrade.source_config(migration_id)` | [Upgrade](Workflows.md#upgrade) [Linked Source Data Migration](#linked-source-data-migration) | **No** | `upgrade.linked_source(migration_id)` | [Upgrade](Workflows.md#upgrade) @@ -313,6 +316,52 @@ def linked_source_size(direct_source, repository, source_config): return database_size ``` +## Direct Linked Source to Physical + +Converts a [Direct Linked Source](Glossary.md#direct-linking) to a physical source. Only applies when a physical export is requested on a dSource using a [Direct Linking](Glossary.md#direct-linking) strategy. + +### Required / Optional +**Optional.** + +### Delphix Engine Operations + +* [Linked Source to Physical](Workflows.md#linked-source-to-physical) + +### Signature + +`def direct_source_to_physical(direct_source, repository, source_config, snapshot, physical_source)` + +### Decorator + +`linked.source_to_physical()` + +### Arguments + +Argument | Type | Description +-------- | ---- | ----------- +direct_source | [DirectSource](Classes.md#directsource) | The direct linked source being exported. +repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source. +source_config | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) | The source config associated with this source. +snapshot | [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-class) | The snapshot being exported. +physical_source | [PhysicalSource](Classes.md#physicalsource) | The target physical source, including connection, target directory, and user-defined parameters from the [VirtualToPhysicalDefinition](Schemas_and_Autogenerated_Classes.md#virtualtophysicaldefinition-class). + +### Returns +None + +### Example + +```python +from dlpx.virtualization.platform import Plugin + +plugin = Plugin() + +@plugin.linked.source_to_physical() +def direct_source_to_physical(direct_source, repository, source_config, snapshot, physical_source): + target_dir = physical_source.target_directory + # Copy or export data to target_dir on the physical host + pass +``` + ## Staged Linked Source Pre-Snapshot Sets up a [dSource](Glossary.md#dsource) to ingest data. Only applies when using a [Staged Linking](Glossary.md#staged-linking) strategy. @@ -687,6 +736,52 @@ def linked_source_size(staged_source, repository, source_config): return database_size ``` +## Staged Linked Source to Physical + +Converts a [Staged Linked Source](Glossary.md#staged-linking) to a physical source. Only applies when a physical export is requested on a dSource using a [Staged Linking](Glossary.md#staged-linking) strategy. + +### Required / Optional +**Optional.** + +### Delphix Engine Operations + +* [Linked Source to Physical](Workflows.md#linked-source-to-physical) + +### Signature + +`def staged_source_to_physical(staged_source, repository, source_config, snapshot, physical_source)` + +### Decorator + +`linked.source_to_physical()` + +### Arguments + +Argument | Type | Description +-------- | ---- | ----------- +staged_source | [StagedSource](Classes.md#stagedsource) | The staged linked source being exported. +repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source. +source_config | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) | The source config associated with this source. +snapshot | [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-class) | The snapshot being exported. +physical_source | [PhysicalSource](Classes.md#physicalsource) | The target physical source, including connection, target directory, and user-defined parameters from the [VirtualToPhysicalDefinition](Schemas_and_Autogenerated_Classes.md#virtualtophysicaldefinition-class). + +### Returns +None + +### Example + +```python +from dlpx.virtualization.platform import Plugin + +plugin = Plugin() + +@plugin.linked.source_to_physical() +def staged_source_to_physical(staged_source, repository, source_config, snapshot, physical_source): + target_dir = physical_source.target_directory + # Copy or export data to target_dir on the physical host + pass +``` + ## Virtual Source Initialize Initializes a brand-new [empty VDB](Glossary.md#empty-vdb). As with all VDBs, this new dataset will have access to mounted Delphix Engine storage, but of course there will be no data there at first. @@ -1289,6 +1384,52 @@ def virtual_source_size(virtual_source, repository, source_config): ``` +## Virtual Source to Physical + +Converts a [Virtual Source](Glossary.md#virtual-source) to a physical source. Only applies when a physical export is requested on a VDB. + +### Required / Optional +**Optional.** + +### Delphix Engine Operations + +* [Virtual Source to Physical](Workflows.md#virtual-source-to-physical) + +### Signature + +`def virtual_source_to_physical(virtual_source, repository, source_config, snapshot, physical_source)` + +### Decorator + +`virtual.source_to_physical()` + +### Arguments + +Argument | Type | Description +-------- | ---- | ----------- +virtual_source | [VirtualSource](Classes.md#virtualsource) | The virtual source being exported. +repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source. +source_config | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) | The source config associated with this source. +snapshot | [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-class) | The snapshot being exported. +physical_source | [PhysicalSource](Classes.md#physicalsource) | The target physical source, including connection, target directory, and user-defined parameters from the [VirtualToPhysicalDefinition](Schemas_and_Autogenerated_Classes.md#virtualtophysicaldefinition-class). + +### Returns +None + +### Example + +```python +from dlpx.virtualization.platform import Plugin + +plugin = Plugin() + +@plugin.virtual.source_to_physical() +def virtual_source_to_physical(virtual_source, repository, source_config, snapshot, physical_source): + target_dir = physical_source.target_directory + # Copy or export data to target_dir on the physical host + pass +``` + ## Repository Data Migration A Repository [Data Migration](Glossary.md#data-migration) migrates repository data from an older [schema](Glossary.md#schema) format to an updated schema format. diff --git a/docs/docs/References/Schemas.md b/docs/docs/References/Schemas.md index 2406cd8a..745d867f 100644 --- a/docs/docs/References/Schemas.md +++ b/docs/docs/References/Schemas.md @@ -13,7 +13,7 @@ Plugins use [schemas](Glossary.md#schema) to describe the format of such data. O 2. It is used to autogenerate a custom user interface, and to validate user inputs. 3. It is used to [autogenerate Python classes](Schemas_and_Autogenerated_Classes.md) that can be used by plugin code to access and manipulate user input and stored data. -There are five plugin-customizable data formats: +There are seven plugin-customizable data formats: Delphix Object | Schema | Autogenerated Class -------------- | ------ | ------------------- @@ -21,8 +21,9 @@ Delphix Object | Schema | Autogenerated Class [Source Config](Glossary.md#source-config) | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-schema) | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) [Linked Source](Glossary.md#linked-source) | [LinkedSourceDefinition](Schemas_and_Autogenerated_Classes.md#linkedsourcedefinition-schema) | [LinkedSourceDefinition](Schemas_and_Autogenerated_Classes.md#linkedsourcedefinition-class) [Virtual Source](Glossary.md#virtual-source) | [VirtualSourceDefinition](Schemas_and_Autogenerated_Classes.md#virtualsourcedefinition-schema)| [VirtualSourceDefinition](Schemas_and_Autogenerated_Classes.md#virtualsourcedefinition-class) -[Snapshot](Glossary.md#linked-source) | [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-schema) | [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-class) +[Snapshot](Glossary.md#snapshot) | [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-schema) | [SnapshotDefinition](Schemas_and_Autogenerated_Classes.md#snapshotdefinition-class) [Snapshot Parameters](Glossary.md#snapshot-parameters) | [SnapshotParametersDefinition](Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-schema) | [SnapshotParametersDefinition](Schemas_and_Autogenerated_Classes.md#snapshotparametersdefinition-class) +[Physical Source](Glossary.md#physical-source) | [VirtualToPhysicalDefinition](Schemas_and_Autogenerated_Classes.md#virtualtophysicaldefinition-schema) | [VirtualToPhysicalDefinition](Schemas_and_Autogenerated_Classes.md#virtualtophysicaldefinition-class) !!! warning There are limits to how much data can be stored within a plugin defined schema. See [Message Limits](../Best_Practices/Message_Limits.md) for details. diff --git a/docs/docs/References/Schemas_and_Autogenerated_Classes.md b/docs/docs/References/Schemas_and_Autogenerated_Classes.md index d396d6e6..4c4dc22d 100644 --- a/docs/docs/References/Schemas_and_Autogenerated_Classes.md +++ b/docs/docs/References/Schemas_and_Autogenerated_Classes.md @@ -295,3 +295,44 @@ from generated.defintions import SnapshotParametersDefinition # Since "resync" is required, it must be specified when constructing the object snapshot_parameter_definition = SnapshotParametersDefinition(resync=True) ``` + +## VirtualToPhysicalDefinition + +Defines properties for the target physical source when converting a virtual or linked source to a physical source. + +### VirtualToPhysicalDefinition Schema + +The `virtualToPhysicalDefinition` is a **required** schema in `schema.json`. It can be an empty object (`{}`) if no additional user input is needed for the physical conversion operation. + +```json +{ + "type": "object", + "additionalProperties": false, + "properties": { + "targetDirectory": { "type": "string" } + } +} +``` + +### VirtualToPhysicalDefinition Class + +Autogenerated based on the [VirtualToPhysicalDefinition Schema](#virtualtophysicaldefinition-schema). + +```python +class VirtualToPhysicalDefinition: + + def __init__(self, target_directory=None): + self._inner_dict = {"targetDirectory": target_directory} +``` + +> To use the class: + +```python +from generated.definitions import VirtualToPhysicalDefinition + +# Construct with an optional target directory +vtop = VirtualToPhysicalDefinition(target_directory="/mnt/physical_target") + +# Access the property +target_dir = vtop.target_directory +``` diff --git a/docs/docs/References/Version_Compatibility.md b/docs/docs/References/Version_Compatibility.md index 0507dc89..0af640b6 100644 --- a/docs/docs/References/Version_Compatibility.md +++ b/docs/docs/References/Version_Compatibility.md @@ -4,6 +4,7 @@ | vSDK Version | Earliest Supported DE Version | Latest Supported DE Version | |------------------------------------------|:-----------------------------:|:-----------------------------------------------------------------:| +| [5.1.0](../Release_Notes/5.1.0/5.1.0.md) | 2026.4.0.0 | [Latest Release](https://cd.delphix.com/docs/latest/new-features) | | [5.0.1](../Release_Notes/5.0.1/5.0.1.md) | 2025.2.0.0 | [Latest Release](https://cd.delphix.com/docs/latest/new-features) | | [5.0.0](../Release_Notes/5.0.0/5.0.0.md) | 29.0.0.0 | [Latest Release](https://cd.delphix.com/docs/latest/new-features) | | [4.1.0](../Release_Notes/4.1.0/4.1.0.md) | 12.0.0.0 | [Latest Release](https://cd.delphix.com/docs/latest/new-features) | @@ -20,6 +21,7 @@ | vSDK Version | Python Version | |------------------------------------------|:--------------:| +| [5.1.0](../Release_Notes/5.1.0/5.1.0.md) | 3.11 | | [5.0.1](../Release_Notes/5.0.1/5.0.1.md) | 3.11 | | [5.0.0](../Release_Notes/5.0.0/5.0.0.md) | 3.11 | | [4.1.0](../Release_Notes/4.1.0/4.1.0.md) | 3.8 | diff --git a/docs/docs/References/Workflows.md b/docs/docs/References/Workflows.md index d58ed815..6193d823 100644 --- a/docs/docs/References/Workflows.md +++ b/docs/docs/References/Workflows.md @@ -24,6 +24,10 @@ ![Screenshot](images/LinkedSourceDelete.png) +## Linked Source to Physical + +![Screenshot](images/LinkedSourceToPhysical.png) + ## Virtual Source Provision ![Screenshot](images/VirtualSourceProvision.png) @@ -68,6 +72,10 @@ Note: pre- and post-start hooks are only run if `extendedStartStopHooks` is set Note: pre- and post-stop hooks are only run if `extendedStartStopHooks` is set to `true` in the [plugin config](Plugin_Config.md). +## Virtual Source to Physical + +![Screenshot](images/VirtualSourceToPhysical.png) + ## Upgrade ![Screenshot](images/PluginUpgrade.png) diff --git a/docs/docs/References/html/LinkedSourceToPhysical.html b/docs/docs/References/html/LinkedSourceToPhysical.html new file mode 100644 index 00000000..ce8ff82e --- /dev/null +++ b/docs/docs/References/html/LinkedSourceToPhysical.html @@ -0,0 +1,11 @@ + + + + +Untitled Diagram + + +
+ + + \ No newline at end of file diff --git a/docs/docs/References/html/VirtualSourceToPhysical.html b/docs/docs/References/html/VirtualSourceToPhysical.html new file mode 100644 index 00000000..452864d5 --- /dev/null +++ b/docs/docs/References/html/VirtualSourceToPhysical.html @@ -0,0 +1,11 @@ + + + + +Untitled Diagram + + +
+ + + \ No newline at end of file diff --git a/docs/docs/References/images/LinkedSourceToPhysical.png b/docs/docs/References/images/LinkedSourceToPhysical.png new file mode 100644 index 00000000..9007ad7f Binary files /dev/null and b/docs/docs/References/images/LinkedSourceToPhysical.png differ diff --git a/docs/docs/References/images/VirtualSourceToPhysical.png b/docs/docs/References/images/VirtualSourceToPhysical.png new file mode 100644 index 00000000..5d2c0dcc Binary files /dev/null and b/docs/docs/References/images/VirtualSourceToPhysical.png differ diff --git a/docs/docs/Release_Notes/.pages b/docs/docs/Release_Notes/.pages index 17365274..d7881402 100644 --- a/docs/docs/Release_Notes/.pages +++ b/docs/docs/Release_Notes/.pages @@ -1,4 +1,5 @@ arrange: + - 5.1.0 - 5.0.1 - 5.0.0 - 4.1.0 diff --git a/docs/docs/Release_Notes/5.1.0/.pages b/docs/docs/Release_Notes/5.1.0/.pages new file mode 100644 index 00000000..f343bb77 --- /dev/null +++ b/docs/docs/Release_Notes/5.1.0/.pages @@ -0,0 +1,3 @@ +arrange: + - 5.1.0.md + - 5.1.0_Breaking_Changes.md diff --git a/docs/docs/Release_Notes/5.1.0/5.1.0.md b/docs/docs/Release_Notes/5.1.0/5.1.0.md new file mode 100644 index 00000000..7072540c --- /dev/null +++ b/docs/docs/Release_Notes/5.1.0/5.1.0.md @@ -0,0 +1,14 @@ +# Release - v5.1.0 + +To install or upgrade the SDK, refer to instructions [here](../../Getting_Started.md#installation). + +## New & Improved + +* Added support for converting virtual and linked sources to physical sources via the new `source_to_physical` operation. + * Use `@plugin.virtual.source_to_physical()` to implement physical export for VDBs. See [Virtual Source to Physical](../../References/Plugin_Operations.md#virtual-source-to-physical). + * Use `@plugin.linked.source_to_physical()` to implement physical export for dSources (both direct and staged linking). See [Direct Linked Source to Physical](../../References/Plugin_Operations.md#direct-linked-source-to-physical) and [Staged Linked Source to Physical](../../References/Plugin_Operations.md#staged-linked-source-to-physical). + * A new [PhysicalSource](../../References/Classes.md#physicalsource) class is passed to the operation with the target connection, directory, and user-defined parameters. + +## Breaking Changes + +* A new required schema `virtualToPhysicalDefinition` must be defined in `schema.json`. See [VirtualToPhysicalDefinition](../../References/Schemas_and_Autogenerated_Classes.md#virtualtophysicaldefinition). diff --git a/docs/docs/Release_Notes/5.1.0/5.1.0_Breaking_Changes.md b/docs/docs/Release_Notes/5.1.0/5.1.0_Breaking_Changes.md new file mode 100644 index 00000000..8496a646 --- /dev/null +++ b/docs/docs/Release_Notes/5.1.0/5.1.0_Breaking_Changes.md @@ -0,0 +1,6 @@ +# Breaking Changes - v.5.1.0 + +## New Required Schema: virtualToPhysicalDefinition +The `schema.json` must now define a `virtualToPhysicalDefinition` schema. This schema defines the user-facing parameters for the `source_to_physical` operation. + +See [VirtualToPhysicalDefinition](../../References/Schemas_and_Autogenerated_Classes.md#virtualtophysicaldefinition) for details on how to define this schema. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 35f13b30..cfb23989 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: Delphix Virtualization SDK 5.0.1 +site_name: Delphix Virtualization SDK 5.1.0 repo_name: Delphix Virtualization SDK repo_url: https://github.com/delphix/virtualization-sdk/ @@ -20,7 +20,7 @@ theme: - content.code.copy - content.code.annotate -copyright: Copyright © 2025 Delphix Corp. +copyright: Copyright © 2026 Delphix Corp. extra: analytics: diff --git a/libs/setup.py b/libs/setup.py index 7d39c5be..4f69b703 100644 --- a/libs/setup.py +++ b/libs/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.9.0", + "dvp-api == 1.10.0.dev0", "dvp-common == {}".format(version), "six >= 1.17, < 1.18", ] diff --git a/platform/setup.py b/platform/setup.py index a63f91e8..3377837b 100644 --- a/platform/setup.py +++ b/platform/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.9.0", + "dvp-api == 1.10.0.dev0", "dvp-common == {}".format(version), "six >= 1.17, < 1.18", ] diff --git a/platform/src/main/python/dlpx/virtualization/platform/_linked.py b/platform/src/main/python/dlpx/virtualization/platform/_linked.py index eda09add..dbc2a94c 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_linked.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_linked.py @@ -13,7 +13,7 @@ from dlpx.virtualization.common.exceptions import PluginRuntimeError from dlpx.virtualization.platform import (DirectSource, Mount, MountSpecification, StagedSource, - Status) + Status, PhysicalSource) from dlpx.virtualization.platform import validation_util as v from dlpx.virtualization.platform.exceptions import ( IncorrectReturnTypeError, OperationAlreadyDefinedError, @@ -33,6 +33,7 @@ def __init__(self): self.worker_impl = None self.mount_specification_impl = None self.source_size_impl = None + self.source_to_physical_impl = None def pre_snapshot(self): def pre_snapshot_decorator(pre_snapshot_impl): @@ -112,6 +113,16 @@ def source_size_decorator(source_size_impl): return source_size_decorator + def source_to_physical(self): + def source_to_physical_decorator(source_to_physical_impl): + if self.source_to_physical_impl: + raise OperationAlreadyDefinedError(Op.LINKED_SOURCE_TO_PHYSICAL) + self.source_to_physical_impl = v.check_function( + source_to_physical_impl, Op.LINKED_SOURCE_TO_PHYSICAL) + return source_to_physical_impl + + return source_to_physical_decorator + @staticmethod def _from_protobuf_remote_mount(remote_mount): return Mount(remote_environment=RemoteEnvironment.from_proto( @@ -323,6 +334,72 @@ def _internal_direct_source_size(self, request): return direct_source_size_response + def _internal_direct_source_to_physical(self, request): + """Direct Source to Physical Wrapper for direct plugins. + + Executed as part of several operations to convert a virtual source + to a physical source for a direct source. + + Run virtual_to_physical operation for a direct source. + + Args: + request (DirectSourceToPhysicalRequest): Virtual to Physical Request + arguments. + + Returns: + DirectSourceToPhysicalResponse: A response containing the return value - + DirectSourceToPhysicalResult. In case of errors, response object will + contain PluginErrorResult. + """ + # Reasoning for method imports are in this file's docstring. + from generated.definitions import RepositoryDefinition + from generated.definitions import LinkedSourceDefinition + from generated.definitions import SourceConfigDefinition + from generated.definitions import SnapshotDefinition + from generated.definitions import VirtualToPhysicalDefinition + + # + # While linked.virtual_to_physical() is not a required operation, + # this should not be called if it wasn't implemented. + # + if not self.source_to_physical_impl: + raise OperationNotDefinedError(Op.LINKED_SOURCE_TO_PHYSICAL) + + direct_source_definition = LinkedSourceDefinition.from_dict( + json.loads(request.direct_source.linked_source.parameters.json)) + direct_source = DirectSource( + guid=request.direct_source.linked_source.guid, + connection=RemoteConnection.from_proto( + request.direct_source.connection), + parameters=direct_source_definition) + virtual_to_physical_source_definition = VirtualToPhysicalDefinition.from_dict( + json.loads(request.physical_source.parameters.json)) + physical_source = PhysicalSource( + guid=request.physical_source.guid, + connection=RemoteConnection.from_proto(request.physical_source.connection), + target_directory=request.physical_source.target_directory, + parameters=virtual_to_physical_source_definition) + + repository = RepositoryDefinition.from_dict( + json.loads(request.repository.parameters.json)) + source_config = SourceConfigDefinition.from_dict( + json.loads(request.source_config.parameters.json)) + snapshot = SnapshotDefinition.from_dict( + json.loads(request.snapshot.parameters.json)) + + self.source_to_physical_impl( + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + direct_source_to_physical_response = ( + platform_pb2.DirectSourceToPhysicalResponse()) + direct_source_to_physical_response.return_value.CopyFrom( + platform_pb2.DirectSourceToPhysicalResult()) + return direct_source_to_physical_response + def _internal_staged_pre_snapshot(self, request): """Pre Snapshot Wrapper for staged plugins. @@ -852,3 +929,76 @@ def _internal_staged_source_size(self, request): staged_source_size_response.return_value.database_size = source_size return staged_source_size_response + + def _internal_staged_source_to_physical(self, request): + """Staged Source to Physical Wrapper for staged plugins. + + Executed as part of several operations to convert a virtual source + to a physical source for a staged source. + + Run virtual_to_physical operation for a staged source. + + Args: + request (StagedSourceToPhysicalRequest): Virtual to Physical Request + arguments. + + Returns: + StagedSourceToPhysicalResponse: A response containing the return value - + StagedSourceToPhysicalResult. In case of errors, response object will + contain PluginErrorResult. + """ + # Reasoning for method imports are in this file's docstring. + from generated.definitions import RepositoryDefinition + from generated.definitions import LinkedSourceDefinition + from generated.definitions import SourceConfigDefinition + from generated.definitions import SnapshotDefinition + from generated.definitions import VirtualToPhysicalDefinition + + # + # While linked.virtual_to_physical() is not a required operation, + # this should not be called if it wasn't implemented. + # + if not self.source_to_physical_impl: + raise OperationNotDefinedError(Op.LINKED_SOURCE_TO_PHYSICAL) + + staged_source_definition = LinkedSourceDefinition.from_dict( + json.loads(request.staged_source.linked_source.parameters.json)) + staged_mount, mounts = LinkedOperations._get_mounts_from_request(request) + staged_source = StagedSource( + guid=request.staged_source.linked_source.guid, + source_connection=RemoteConnection.from_proto( + request.staged_source.source_connection), + parameters=staged_source_definition, + mount=staged_mount, + staged_connection=RemoteConnection.from_proto( + request.staged_source.staged_connection), + mounts=mounts) + + virtual_to_physical_source_definition = VirtualToPhysicalDefinition.from_dict( + json.loads(request.physical_source.parameters.json)) + physical_source = PhysicalSource( + guid=request.physical_source.guid, + connection=RemoteConnection.from_proto(request.physical_source.connection), + target_directory=request.physical_source.target_directory, + parameters=virtual_to_physical_source_definition) + + repository = RepositoryDefinition.from_dict( + json.loads(request.repository.parameters.json)) + source_config = SourceConfigDefinition.from_dict( + json.loads(request.source_config.parameters.json)) + snapshot = SnapshotDefinition.from_dict( + json.loads(request.snapshot.parameters.json)) + + self.source_to_physical_impl( + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source + ) + + staged_source_to_physical_response = ( + platform_pb2.StagedSourceToPhysicalResponse()) + staged_source_to_physical_response.return_value.CopyFrom( + platform_pb2.StagedSourceToPhysicalResult()) + return staged_source_to_physical_response diff --git a/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py b/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py index bc9cf47a..e26f8046 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_plugin_classes.py @@ -20,8 +20,9 @@ of the autogenerated classes from the schemas (e.g. VirtualSourceDefinition) """ __all__ = [ - "VirtualSource", "StagedSource", "DirectSource", "Status", "MigrationType", - "Mount", "OwnershipSpecification", "MountSpecification" + "VirtualSource", "StagedSource", "DirectSource", "PhysicalSource", + "Status", "MigrationType", "Mount", "OwnershipSpecification", + "MountSpecification" ] @@ -137,6 +138,44 @@ def parameters(self): return self._parameters +class PhysicalSource(object): + def __init__(self, guid, connection, target_directory, parameters): + self._guid = guid + if not isinstance(connection, RemoteConnection): + raise IncorrectTypeError(RemoteConnection, 'connection', + type(connection), RemoteConnection) + self._connection = connection + if not isinstance(target_directory, six.string_types): + raise IncorrectTypeError(PhysicalSource, 'target_directory', + type(target_directory), + six.string_types[0]) + self._target_directory = target_directory + self._parameters = parameters + + @property + def guid(self): + """str: The unique guid identifier for this PhysicalSource.""" + return self._guid + + @property + def connection(self): + """RemoteConnection: The RemoteConnection for this + PhysicalSource.""" + return self._connection + + @property + def target_directory(self): + """str: The target directory path for this PhysicalSource.""" + return self._target_directory + + @property + def parameters(self): + """PhysicalSourceDefinition: The PhysicalSourceDefinition for this + PhysicalSource. + """ + return self._parameters + + class Status(Enum): ACTIVE = 0 INACTIVE = 1 diff --git a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py index 05fa5cd2..f947efad 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py @@ -11,7 +11,7 @@ from dlpx.virtualization.api import common_pb2, platform_pb2 from dlpx.virtualization.common import RemoteConnection, RemoteEnvironment from dlpx.virtualization.platform import (Mount, MountSpecification, Status, - VirtualSource) + VirtualSource, PhysicalSource) from dlpx.virtualization.platform import validation_util as v from dlpx.virtualization.platform.exceptions import ( IncorrectReturnTypeError, OperationAlreadyDefinedError, @@ -35,6 +35,7 @@ def __init__(self): self.initialize_impl = None self.mount_specification_impl = None self.source_size_impl = None + self.source_to_physical_impl = None def configure(self): def configure_decorator(configure_impl): @@ -153,6 +154,17 @@ def source_size_decorator(source_size_impl): return source_size_decorator + def source_to_physical(self): + def source_to_physical_decorator(source_to_physical_impl): + if self.source_to_physical_impl: + raise OperationAlreadyDefinedError( + Op.VIRTUAL_SOURCE_TO_PHYSICAL) + self.source_to_physical_impl = v.check_function( + source_to_physical_impl, Op.VIRTUAL_SOURCE_TO_PHYSICAL) + return source_to_physical_impl + + return source_to_physical_decorator + @staticmethod def _from_protobuf_single_subset_mount(single_subset_mount): return Mount(remote_environment=RemoteEnvironment.from_proto( @@ -848,3 +860,72 @@ def _internal_virtual_source_size(self, request): virtual_source_size_response.return_value.database_size = source_size return virtual_source_size_response + + def _internal_virtual_source_to_physical(self, request): + """Virtual to Physical Wrapper. + + Executed as part of several operations to convert a virtual source + to a physical source. + + Run virtual_to_physical operation for a virtual source. + + Args: + request (VirtualSourceToPhysicalRequest): Virtual to Physical Request + arguments. + + Returns: + VirtualSourceToPhysicalResponse: A response containing the return value - + VirtualSourceToPhysicalResult. In case of errors, response object will + contain PluginErrorResult. + """ + # Reasoning for method imports are in this file's docstring. + from generated.definitions import VirtualSourceDefinition + from generated.definitions import RepositoryDefinition + from generated.definitions import SourceConfigDefinition + from generated.definitions import SnapshotDefinition + from generated.definitions import VirtualToPhysicalDefinition + + # + # While virtual.virtual_to_physical() is not a required operation, + # this should not be called if it wasn't implemented. + # + if not self.source_to_physical_impl: + raise OperationNotDefinedError(Op.VIRTUAL_SOURCE_TO_PHYSICAL) + + virtual_source_definition = VirtualSourceDefinition.from_dict( + json.loads(request.virtual_source.parameters.json)) + mounts = [ + VirtualOperations._from_protobuf_single_subset_mount(m) + for m in request.virtual_source.mounts + ] + virtual_source = VirtualSource(guid=request.virtual_source.guid, + connection=RemoteConnection.from_proto( + request.virtual_source.connection), + parameters=virtual_source_definition, + mounts=mounts) + virtual_to_physical_source_definition = VirtualToPhysicalDefinition.from_dict( + json.loads(request.physical_source.parameters.json)) + physical_source = PhysicalSource( + guid=request.physical_source.guid, + connection=RemoteConnection.from_proto(request.physical_source.connection), + target_directory=request.physical_source.target_directory, + parameters=virtual_to_physical_source_definition) + repository = RepositoryDefinition.from_dict( + json.loads(request.repository.parameters.json)) + source_config = SourceConfigDefinition.from_dict( + json.loads(request.source_config.parameters.json)) + snapshot = SnapshotDefinition.from_dict( + json.loads(request.snapshot.parameters.json)) + + self.source_to_physical_impl( + virtual_source=virtual_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + virtual_to_physical_response = ( + platform_pb2.VirtualSourceToPhysicalResponse()) + virtual_to_physical_response.return_value.CopyFrom( + platform_pb2.VirtualSourceToPhysicalResult()) + return virtual_to_physical_response diff --git a/platform/src/main/python/dlpx/virtualization/platform/operation.py b/platform/src/main/python/dlpx/virtualization/platform/operation.py index 6d033d21..8eeeef5f 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/operation.py +++ b/platform/src/main/python/dlpx/virtualization/platform/operation.py @@ -18,6 +18,7 @@ class Operation(Enum): LINKED_WORKER = 'linked.worker()' LINKED_MOUNT_SPEC = 'linked.mount_specification()' LINKED_SOURCE_SIZE = 'linked.source_size()' + LINKED_SOURCE_TO_PHYSICAL = 'linked.source_to_physical()' VIRTUAL_CONFIGURE = 'virtual.configure()' VIRTUAL_UNCONFIGURE = 'virtual.unconfigure()' @@ -31,6 +32,7 @@ class Operation(Enum): VIRTUAL_INITIALIZE = 'virtual.initialize()' VIRTUAL_MOUNT_SPEC = 'virtual.mount_specification()' VIRTUAL_SOURCE_SIZE = 'virtual.source_size()' + VIRTUAL_SOURCE_TO_PHYSICAL = 'virtual.source_to_physical()' UPGRADE_REPOSITORY = 'upgrade.repository()' UPGRADE_SOURCE_CONFIG = 'upgrade.source_config()' diff --git a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py index e4ef3419..9b5d087c 100644 --- a/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py +++ b/platform/src/test/python/dlpx/virtualization/fake_generated_definitions.py @@ -120,3 +120,22 @@ def from_dict(input_dict): def to_dict(self): return {"resync": self._resync} + + +class VirtualToPhysicalDefinition(Model): + def __init__(self, name): + self.swagger_types = {'name': six.string_types[0]} + + self.attribute_map = {'name': 'name'} + self._name = name + + @property + def name(self): + return self._name + + @staticmethod + def from_dict(input_dict): + return VirtualToPhysicalDefinition(input_dict['name']) + + def to_dict(self): + return {"name": self._name} diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin.py b/platform/src/test/python/dlpx/virtualization/test_plugin.py index f687549e..fb7adbf5 100755 --- a/platform/src/test/python/dlpx/virtualization/test_plugin.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin.py @@ -39,6 +39,7 @@ TEST_DIRECT_SOURCE = 'TestDirectSource' TEST_STAGED_SOURCE = 'TestStagedSource' TEST_VIRTUAL_SOURCE = 'TestVirtualSource' +TEST_PHYSICAL_SOURCE = 'TestPhysicalSource' # This is a simple JSON object that has only "name" property defined. SIMPLE_JSON = '{{"name": "{0}"}}' @@ -49,6 +50,7 @@ TEST_DIRECT_SOURCE_JSON = SIMPLE_JSON.format(TEST_DIRECT_SOURCE) TEST_STAGED_SOURCE_JSON = SIMPLE_JSON.format(TEST_STAGED_SOURCE) TEST_VIRTUAL_SOURCE_JSON = SIMPLE_JSON.format(TEST_VIRTUAL_SOURCE) +TEST_PHYSICAL_SOURCE_JSON = SIMPLE_JSON.format(TEST_PHYSICAL_SOURCE) TEST_SNAPSHOT_PARAMS_JSON = '{"resync": false}' TEST_PRE_UPGRADE_PARAMS = {'obj': json.dumps({'name': 'upgrade'})} TEST_POST_MIGRATION_METADATA_1 = (json.dumps( @@ -157,6 +159,10 @@ def assert_mounts(mounts): assert len(mounts) == 1 TestPlugin.assert_mount(mounts[0]) + @staticmethod + def assert_target_directory(target_directory): + assert target_directory == TEST_BINARY_PATH + @staticmethod def assert_virtual_source(virtual_source): assert virtual_source.guid == TEST_GUID @@ -204,6 +210,13 @@ def assert_snapshot(snapshot): def assert_snapshot_parameters(snapshot_parameters): assert not snapshot_parameters.resync + @staticmethod + def assert_physical_source(physical_source): + assert physical_source.guid == TEST_GUID + TestPlugin.assert_connection(physical_source.connection) + TestPlugin.assert_target_directory(physical_source.target_directory) + assert physical_source.parameters.name == TEST_PHYSICAL_SOURCE + @staticmethod @pytest.fixture def host(): @@ -232,6 +245,11 @@ def mount(environment): mount.shared_path = TEST_SHARED_PATH return mount + @staticmethod + @pytest.fixture + def target_directory(): + return TEST_BINARY_PATH + @staticmethod @pytest.fixture def user(): @@ -266,6 +284,16 @@ def virtual_source(connection, mount): virtual_source.mounts.extend([mount]) return virtual_source + @staticmethod + @pytest.fixture + def physical_source(connection, target_directory): + physical_source = common_pb2.PhysicalSource() + physical_source.guid = TEST_GUID + physical_source.connection.CopyFrom(connection) + physical_source.parameters.json = TEST_PHYSICAL_SOURCE_JSON + physical_source.target_directory = target_directory + return physical_source + @staticmethod @pytest.fixture def repository(): @@ -342,7 +370,8 @@ def setup_request(request, repository=None, source_config=None, snapshot=None, - snapshot_parameters=None): + snapshot_parameters=None, + physical_source=None): if virtual_source: request.virtual_source.CopyFrom(virtual_source) @@ -364,6 +393,9 @@ def setup_request(request, if snapshot_parameters: request.snapshot_parameters.CopyFrom(snapshot_parameters) + if physical_source: + request.physical_source.CopyFrom(physical_source) + @staticmethod def assert_plugin_args(**kwargs): for key, value in kwargs.items(): @@ -391,6 +423,9 @@ def assert_plugin_args(**kwargs): TestPlugin.assert_snapshot_parameters( kwargs['snapshot_parameters']) + if 'physical_source' in kwargs: + TestPlugin.assert_physical_source(kwargs['physical_source']) + @staticmethod def test_virtual_configure(my_plugin, virtual_source, repository, snapshot): @@ -792,6 +827,35 @@ def virtual_source_size_impl(virtual_source, repository, source_config): assert virtual_source_size_response.return_value.database_size == 100 + @staticmethod + def test_virtual_source_to_physical(my_plugin, virtual_source, repository, + source_config, snapshot, physical_source): + + @my_plugin.virtual.source_to_physical() + def virtual_source_to_physical_impl(virtual_source, repository, + source_config, snapshot, physical_source): + TestPlugin.assert_plugin_args(virtual_source=virtual_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + virtual_to_physical_request = platform_pb2.VirtualSourceToPhysicalRequest() + TestPlugin.setup_request(request=virtual_to_physical_request, + virtual_source=virtual_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + virtual_to_physical_response = my_plugin.virtual. \ + _internal_virtual_source_to_physical(virtual_to_physical_request) + expected_result = platform_pb2.VirtualSourceToPhysicalResult() + # Check that the response's oneof is set to return_value and not error + assert virtual_to_physical_response.WhichOneof( + 'result') == 'return_value' + assert virtual_to_physical_response.return_value == expected_result + @staticmethod def test_repository_discovery(my_plugin, connection): @my_plugin.discovery.repository() @@ -1009,6 +1073,35 @@ def direct_source_size_impl(direct_source, repository, source_config): assert direct_source_size_response.return_value.database_size == 100 + @staticmethod + def test_direct_source_to_physical(my_plugin, direct_source, repository, + source_config, snapshot, physical_source): + + @my_plugin.linked.source_to_physical() + def direct_source_to_physical_impl(direct_source, repository, + source_config, snapshot, physical_source): + TestPlugin.assert_plugin_args(direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + direct_to_physical_request = platform_pb2.DirectSourceToPhysicalRequest() + TestPlugin.setup_request(request=direct_to_physical_request, + direct_source=direct_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + direct_to_physical_response = my_plugin.linked. \ + _internal_direct_source_to_physical(direct_to_physical_request) + expected_result = platform_pb2.DirectSourceToPhysicalResult() + # Check that the response's oneof is set to return_value and not error + assert direct_to_physical_response.WhichOneof( + 'result') == 'return_value' + assert direct_to_physical_response.return_value == expected_result + @staticmethod @pytest.mark.parametrize("staged_source", ["mount", "mounts", "Both"], indirect=["staged_source"]) @@ -1249,6 +1342,40 @@ def staged_source_size_impl(staged_source, repository, source_config): if staged_source_size_response: assert staged_source_size_response.return_value.database_size == 0 + @staticmethod + @pytest.mark.parametrize("staged_source", ["mount", "mounts", "Both"], + indirect=["staged_source"]) + def test_staged_source_to_physical(my_plugin, staged_source, repository, + source_config, snapshot, physical_source): + + @my_plugin.linked.source_to_physical() + def staged_source_to_physical_impl(staged_source, repository, + source_config, snapshot, physical_source): + TestPlugin.assert_plugin_args(staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + staged_to_physical_request = platform_pb2.StagedSourceToPhysicalRequest() + TestPlugin.setup_request(request=staged_to_physical_request, + staged_source=staged_source, + repository=repository, + source_config=source_config, + snapshot=snapshot, + physical_source=physical_source) + + staged_to_physical_response = TestPlugin._call_stage_methods( + my_plugin.linked._internal_staged_source_to_physical, + staged_to_physical_request, + TestPlugin._raise_staged_source_both_mounts_exception(staged_source)) + expected_result = platform_pb2.StagedSourceToPhysicalResult() + # Check that the response's oneof is set to return_value and not error + if staged_to_physical_response: + assert staged_to_physical_response.WhichOneof( + 'result') == 'return_value' + assert staged_to_physical_response.return_value == expected_result + @staticmethod @pytest.mark.parametrize("staged_source", ["mount", "mounts", "Both"], indirect=["staged_source"]) diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py index 7da2e8bd..68391641 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/build.py @@ -21,6 +21,7 @@ TYPE = 'Plugin' LOCALE_DEFAULT = 'en-us' VIRTUAL_SOURCE_TYPE = 'PluginVirtualSourceDefinition' +VIRTUAL_TO_PHYSICAL_TYPE = 'PluginVirtualToPhysicalDefinition' DISCOVERY_DEFINITION_TYPE = 'PluginDiscoveryDefinition' STAGED_LINKED_SOURCE_TYPE = 'PluginLinkedStagedSourceDefinition' DIRECT_LINKED_SOURCE_TYPE = 'PluginLinkedDirectSourceDefinition' @@ -243,6 +244,10 @@ def prepare_upload_artifact(plugin_config_content, src_dir, schemas, manifest): 'type': VIRTUAL_SOURCE_TYPE, 'parameters': schemas['virtualSourceDefinition'] }, + 'virtualToPhysicalDefinition': { + 'type': VIRTUAL_TO_PHYSICAL_TYPE, + 'parameters': schemas['virtualToPhysicalDefinition'] + }, 'linkedSourceDefinition': { 'type': get_linked_source_definition_type(plugin_config_content), 'parameters': schemas['linkedSourceDefinition'] diff --git a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json index 8dd02073..7f629d50 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/commands/plugin_template/schema_template.json @@ -22,6 +22,11 @@ "additionalProperties" : false, "properties" : {} }, + "virtualToPhysicalDefinition": { + "type": "object", + "additionalProperties" : false, + "properties" : {} + }, "linkedSourceDefinition": { "type": "object", "additionalProperties" : false, diff --git a/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py b/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py index f569c120..48ff97a4 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py @@ -360,6 +360,8 @@ def _prepare_manifest(entry_point, module_content): bool(plugin_object.linked.mount_specification_impl), 'hasLinkedSourceSize': bool(plugin_object.linked.source_size_impl), + 'hasLinkedSourceToPhysical': + bool(plugin_object.linked.source_to_physical_impl), 'hasVirtualConfigure': bool(plugin_object.virtual.configure_impl), 'hasVirtualUnconfigure': @@ -384,6 +386,8 @@ def _prepare_manifest(entry_point, module_content): bool(plugin_object.virtual.initialize_impl), 'hasVirtualSourceSize': bool(plugin_object.virtual.source_size_impl), + 'hasVirtualSourceToPhysical': + bool(plugin_object.virtual.source_to_physical_impl), 'migrationIdList': plugin_object.upgrade.migration_id_list } diff --git a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg index c80f2be1..21d61d2a 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg +++ b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg @@ -20,7 +20,7 @@ # versions in those packages until they are shipped out of band. # [General] -engine_api_version = 1.11.16 +engine_api_version = 1.11.49 distribution_name = dvp-tools package_author = Delphix namespace_package = dlpx diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml index 9193e581..018da1e8 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml @@ -42,6 +42,12 @@ EXPECTED_STAGED_ARGS_BY_OP: - staged_source - repository - source_config + source_to_physical_impl: + - staged_source + - repository + - source_config + - snapshot + - physical_source VirtualOperations: configure_impl: - virtual_source @@ -90,7 +96,12 @@ EXPECTED_STAGED_ARGS_BY_OP: - virtual_source - repository - source_config - + source_to_physical_impl: + - virtual_source + - repository + - source_config + - snapshot + - physical_source EXPECTED_DIRECT_ARGS_BY_OP: @@ -115,6 +126,12 @@ EXPECTED_DIRECT_ARGS_BY_OP: - direct_source - repository - source_config + source_to_physical_impl: + - direct_source + - repository + - source_config + - snapshot + - physical_source VirtualOperations: configure_impl: - virtual_source @@ -163,6 +180,12 @@ EXPECTED_DIRECT_ARGS_BY_OP: - virtual_source - repository - source_config + source_to_physical_impl: + - virtual_source + - repository + - source_config + - snapshot + - physical_source EXPECTED_UPGRADE_ARGS: _repository_id_to_impl: diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json index 59c39057..ea1141dd 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_schema.json @@ -183,6 +183,7 @@ "repositoryDefinition", "sourceConfigDefinition", "virtualSourceDefinition", + "virtualToPhysicalDefinition", "linkedSourceDefinition", "snapshotDefinition", "snapshotParametersDefinition" @@ -218,6 +219,9 @@ } ] }, + "virtualToPhysicalDefinition": { + "$ref": "#/definitions/jsonSchema" + }, "linkedSourceDefinition": { "allOf": [ { diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py index b26be24e..c2cfc504 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_build.py @@ -679,6 +679,19 @@ def test_plugin_missing_schema_def(mock_generate_python, assert not mock_generate_python.called + @staticmethod + @pytest.mark.parametrize('virtual_to_physical_definition', [None]) + @mock.patch('dlpx.virtualization._internal.codegen.generate_python') + def test_plugin_missing_virtual_to_physical_schema_def( + mock_generate_python, plugin_config_file, artifact_file): + with pytest.raises(exceptions.UserError) as err_info: + build.build(plugin_config_file, artifact_file, False, False) + + message = err_info.value.message + assert "'virtualToPhysicalDefinition' is a required property" in message + + assert not mock_generate_python.called + @staticmethod @mock.patch('dlpx.virtualization._internal.codegen.generate_python') @pytest.mark.parametrize('additional_definition', [{ diff --git a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py index ecdfa4e5..67cee89a 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/commands/test_initialize.py @@ -270,11 +270,12 @@ def test_default_schema_definition(schema_template): ] # - # Validate the linked source, virtual source, and snapshot definitions - # have no properties. + # Validate the linked source, virtual source, virtual to physical, and + # snapshot definitions have no properties. # assert schema_template['linkedSourceDefinition']['properties'] == {} assert schema_template['virtualSourceDefinition']['properties'] == {} + assert schema_template['virtualToPhysicalDefinition']['properties'] == {} assert schema_template['snapshotDefinition']['properties'] == {} @staticmethod diff --git a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py index f940f7cc..e9db3e2d 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py @@ -371,6 +371,7 @@ def post_snapshot(direct_source, repository, source_config): linked.worker_impl = None linked.mount_specification_impl = None linked.source_size_impl = None + linked.source_to_physical_impl = None return linked @@ -412,6 +413,7 @@ def mount_specification(virtual_source, repository): virtual.initialize_impl = None virtual.cleanup_impl = None virtual.source_size_impl = None + virtual.source_to_physical_impl = None return virtual @@ -446,6 +448,7 @@ def plugin_manifest(upgrade_operation): 'hasLinkedWorker': False, 'hasLinkedMountSpecification': False, 'hasLinkedSourceSize': False, + 'hasLinkedSourceToPhysical': False, 'hasVirtualConfigure': True, 'hasVirtualUnconfigure': False, 'hasVirtualReconfigure': True, @@ -459,13 +462,15 @@ def plugin_manifest(upgrade_operation): 'hasInitialize': False, 'migrationIdList': upgrade_operation.migration_id_list, 'hasVirtualCleanup': False, + 'hasVirtualSourceToPhysical': False, } return manifest @pytest.fixture def schema_content(repository_definition, source_config_definition, - virtual_source_definition, linked_source_definition, + virtual_source_definition, virtual_to_physical_definition, + linked_source_definition, snapshot_definition, snapshot_parameters_definition, additional_definition): @@ -480,6 +485,9 @@ def schema_content(repository_definition, source_config_definition, if virtual_source_definition: schema['virtualSourceDefinition'] = virtual_source_definition + if virtual_to_physical_definition: + schema['virtualToPhysicalDefinition'] = virtual_to_physical_definition + if linked_source_definition: schema['linkedSourceDefinition'] = linked_source_definition @@ -541,6 +549,19 @@ def virtual_source_definition(): } +@pytest.fixture +def virtual_to_physical_definition(): + return { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'path': { + 'type': 'string' + } + } + } + + @pytest.fixture def linked_source_definition(): return {'type': 'object', 'additionalProperties': False, 'properties': {}} @@ -649,7 +670,8 @@ def add_symlink_folder_to_src_dir(tmpdir, src_dir): @pytest.fixture def artifact_content(engine_api, virtual_source_definition, - linked_source_definition, discovery_definition, + virtual_to_physical_definition, linked_source_definition, + discovery_definition, snapshot_definition, snapshot_parameters_definition): """ This fixture creates base artifact that was generated from build and @@ -685,6 +707,12 @@ def artifact_content(engine_api, virtual_source_definition, 'parameters': virtual_source_definition, } + if virtual_to_physical_definition: + artifact['virtualToPhysicalDefinition'] = { + 'type': 'PluginVirtualToPhysicalDefinition', + 'parameters': virtual_to_physical_definition, + } + if linked_source_definition: artifact['linkedSourceDefinition'] = { 'type': 'PluginLinkedDirectSourceDefinition', diff --git a/tools/src/test/python/dlpx/virtualization/_internal/engine_version.cfg b/tools/src/test/python/dlpx/virtualization/_internal/engine_version.cfg index d5cb24b4..a3b69e2e 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/engine_version.cfg +++ b/tools/src/test/python/dlpx/virtualization/_internal/engine_version.cfg @@ -11,4 +11,4 @@ # This file also should not be updated manually with feature changes. # [General] -engine_api_version = 1.11.16 +engine_api_version = 1.11.49 diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index ce008ac5..9fe28878 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -8,7 +8,7 @@ from dlpx.virtualization._internal import package_util DVP_VERSION = '5.1.0' -DVP_API_VERSION = '1.9.0' +DVP_API_VERSION = '1.10.0' class TestPackageUtil: