Add Provider Schema support with new schema_ref config option (#40)

* move name property to mapper

* add validation and config

* add schema proxy to explorer

* implemented basics of provider + fixed resolving bug with nested refs

* update library dep

* cleanup from merged PRs

* switch collection to use validators

* add map validators

* upgrade and convert float values

* move function

* add docs

* copywrite!

Author: austinvalle

DESIGN.md

@@ -19,12 +19,38 @@ Users of the generator can adjust their OAS to match these assumptions, or sugge
 ## Determining the OAS Schema to map from operations
 
 ### Provider
-<!-- TODO: Update this when we have the provider schema mapping :) -->
-Currently, there is no option in the OpenAPI generator to define the mapping for a provider schema. The `provider.name` property is directly copied to the Framework IR.
+For generating Provider schema code, the [generator config file](./README.md) defines:
+- `provider.name` - required property, which is directly copied to the Framework IR as the name of the Provider.
+- `provider.schema_ref` - optional property, which is a [JSON schema reference](https://json-schema.org/understanding-json-schema/structuring.html#ref) to an existing schema in your OpenAPI spec, typically in the [`components.schema` section](https://spec.openapis.org/oas/v3.1.0#fixed-fields-5). This will be used to [map](#mapping-oas-schema-to-plugin-framework-types) the Provider's schema to Framework IR.
+```yml
+provider:
+  name: fakeprovider
+  # This schema needs to exist in the OpenAPI spec!
+  schema_ref: '#/components/schemas/fake_provider_schema'
+```
 
 ### Resources
 The [generator config file](./README.md) defines the CRUD (`Create`, `Read`, `Update`, `Delete`) operations for a resource in an OAS. In those operations, the generator will search `Create` and `Read` operations for schemas to map to Framework IR. Multiple schemas will be [deep merged](#deep-merge-of-schemas-resources) and the final result will be the Resource schema represented in Framework IR.
 
+```yml
+resources:
+  fake_thing:
+    # Required
+    create:
+      path: /thing
+      method: POST
+    read:
+      path: /thing/{id}
+      method: GET
+    # Optional (currently, no effect)
+    update:
+      path: /thing
+      method: PUT
+    delete:
+      path: /thing/{id}
+      method: DELETE
+```
+
 #### OAS Schema order (resources)
 - `Create` operation [requestBody](https://spec.openapis.org/oas/v3.1.0#requestBodyObject)
     - `requestBody` is the only schema **required** for resources, if not present will log a warning and skip the resource without mapping.
@@ -48,6 +74,14 @@ All schemas found will be deep merged together, with the `requestBody` schema fr
 ### Data Sources
 The [generator config file](./README.md) defines the `Read` operation for a data source in an OAS. In that operation, the generator will search for a response body schema to map to Framework IR. The response body will be [deep merged](#deep-merge-of-schemas-data-sources) with the query parameters and path parameters of the same `Read` operation and the final result will be the Data Source schema represented in Framework IR.
 
+```yml
+data_sources:
+  fake_thing:
+    read:
+      path: /thing/{id}
+      method: GET
+```
+
 #### OAS Schema order (data sources)
 - `Read` operation [response](https://spec.openapis.org/oas/v3.1.0#responsesObject)
     - `response` is the only schema **required** for data sources, if not present will log a warning and skip the data source without mapping.
@@ -110,6 +144,11 @@ For attributes that don't have additional schema information (`ListAttribute`, `
 
 ### Required, Computed, and Optional
 
+#### Provider
+For the provider, all fields in the provided JSON schema (`provider.schema_ref`) marked as [required](https://json-schema.org/understanding-json-schema/reference/object.html#required-properties) will be mapped as a [Required](https://developer.hashicorp.com/terraform/plugin/framework/handling-data/schemas#required) attribute.
+
+If not required, then the field will be mapped as [Optional](https://developer.hashicorp.com/terraform/plugin/framework/handling-data/schemas#optional).
+
 #### Resources
 For resources, all fields, in the `Create` operation `requestBody` OAS schema, marked as [required](https://json-schema.org/understanding-json-schema/reference/object.html#required-properties) will be mapped as a [Required](https://developer.hashicorp.com/terraform/plugin/framework/handling-data/schemas#required) attribute.
 

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/hashicorp/terraform-plugin-codegen-spec v0.0.0-20230623193314-2a16a9d0b6f3
 	github.com/mattn/go-colorable v0.1.13
 	github.com/mitchellh/cli v1.1.5
-	github.com/pb33f/libopenapi v0.8.5
+	github.com/pb33f/libopenapi v0.9.2
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -35,6 +35,6 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	golang.org/x/crypto v0.7.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/sys v0.7.0 // indirect
 )

go.sum

@@ -101,8 +101,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
 github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
-github.com/pb33f/libopenapi v0.8.5 h1:9wOa/qu7mvuX8rgi/VlrKycS/Y1EaeIZaq3iILitgPI=
-github.com/pb33f/libopenapi v0.8.5/go.mod h1:lvUmCtjgHUGVj6WzN3I5/CS9wkXtyN3Ykjh6ZZP5lrI=
+github.com/pb33f/libopenapi v0.9.2 h1:QFxTgTSmW9mnXhQ+myignBh19ZPkFRxnAvbPnFcesDs=
+github.com/pb33f/libopenapi v0.9.2/go.mod h1:lvUmCtjgHUGVj6WzN3I5/CS9wkXtyN3Ykjh6ZZP5lrI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -160,8 +160,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

internal/cmd/generate.go

@@ -15,7 +15,6 @@ import (
 	"github.com/hashicorp/terraform-plugin-codegen-openapi/internal/config"
 	"github.com/hashicorp/terraform-plugin-codegen-openapi/internal/explorer"
 	"github.com/hashicorp/terraform-plugin-codegen-openapi/internal/mapper"
-	"github.com/hashicorp/terraform-plugin-codegen-spec/provider"
 	"github.com/hashicorp/terraform-plugin-codegen-spec/spec"
 
 	"github.com/mitchellh/cli"
@@ -198,10 +197,15 @@ func generateFrameworkIr(dora explorer.Explorer, cfg config.Config) (*spec.Speci
 		return nil, fmt.Errorf("error generating Framework IR for data sources: %w", err)
 	}
 
+	// 6. Use TF info to generate framework IR for provider
+	providerMapper := mapper.NewProviderMapper(explorerProvider, cfg)
+	providerIR, err := providerMapper.MapToIR()
+	if err != nil {
+		return nil, fmt.Errorf("error generating Framework IR for provider: %w", err)
+	}
+
 	return &spec.Specification{
-		Provider: &provider.Provider{
-			Name: explorerProvider.Name,
-		},
+		Provider:    providerIR,
 		Resources:   resourcesIR,
 		DataSources: dataSourcesIR,
 	}, nil

internal/cmd/testdata/edgecase/generated_framework_ir.json

@@ -164,7 +164,54 @@
 		}
 	],
 	"provider": {
-		"name": "edgecase"
+		"name": "edgecase",
+		"schema": {
+			"attributes": [
+				{
+					"name": "bool_prop",
+					"bool": {
+						"optional_required": "optional",
+						"description": "Bool for the provider"
+					}
+				},
+				{
+					"name": "string_prop",
+					"string": {
+						"optional_required": "required",
+						"description": "String for the provider"
+					}
+				},
+				{
+					"name": "triple_nested_map",
+					"list": {
+						"optional_required": "required",
+						"element_type": {
+							"set": {
+								"element_type": {
+									"map": {
+										"element_type": {
+											"object": {
+												"attribute_types": [
+													{
+														"name": "bool_prop",
+														"bool": {}
+													},
+													{
+														"name": "string_prop",
+														"string": {}
+													}
+												]
+											}
+										}
+									}
+								}
+							}
+						},
+						"description": "This list has a set of maps nested underneath!"
+					}
+				}
+			]
+		}
 	},
 	"resources": [
 		{

internal/cmd/testdata/edgecase/openapi_spec.yml

@@ -100,6 +100,21 @@ paths:
                     type: string
 components:
   schemas:
+    edgecase_provider:
+      description: This is the provider schema
+      type: object
+      required:
+        - string_prop
+        - triple_nested_map
+      properties:
+        string_prop:
+          description: String for the provider
+          type: string
+        bool_prop:
+          description: Bool for the provider
+          type: boolean
+        triple_nested_map:
+          $ref: "#/components/schemas/triple_nested_map_schema"
     double_nested_list_schema:
       description: This list has a list nested underneath!
       type: array

internal/cmd/testdata/edgecase/tfopenapigen_config.yml

@@ -1,5 +1,6 @@
 provider:
   name: edgecase
+  schema_ref: '#/components/schemas/edgecase_provider'
 
 resources:
   set_test:

internal/config/parse.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/pb33f/libopenapi/index"
 	"gopkg.in/yaml.v3"
 )
 
@@ -18,7 +19,8 @@ type Config struct {
 }
 
 type Provider struct {
-	Name string `yaml:"name"`
+	Name      string `yaml:"name"`
+	SchemaRef string `yaml:"schema_ref"`
 }
 
 type Resource struct {
@@ -90,6 +92,11 @@ func (p Provider) Validate() error {
 		return errors.New("must have a 'name' property")
 	}
 
+	// All schema refs must be a local, file, or http resolve type
+	if p.SchemaRef != "" && index.DetermineReferenceResolveType(p.SchemaRef) < 0 {
+		return errors.New("'schema_ref' must be a valid JSON schema reference")
+	}
+
 	return nil
 }
 

internal/config/parse_test.go

@@ -106,6 +106,19 @@ func TestParseConfig_Invalid(t *testing.T) {
 			input:            ``,
 			expectedErrRegex: `provider must have a 'name' property`,
 		},
+		"provider - invalid schema_ref - not resolvable": {
+			input: `
+provider:
+  name: example
+  schema_ref: thisaintvalid
+
+data_sources:
+  thing_one:
+    read:
+      path: /example/path/to/thing/{id}
+      method: GET`,
+			expectedErrRegex: `provider 'schema_ref' must be a valid JSON schema reference`,
+		},
 		"at least one resource or data source required": {
 			input: `
 provider:

internal/explorer/config_explorer.go

@@ -4,11 +4,15 @@
 package explorer
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/hashicorp/terraform-plugin-codegen-openapi/internal/config"
 
+	highbase "github.com/pb33f/libopenapi/datamodel/high/base"
 	high "github.com/pb33f/libopenapi/datamodel/high/v3"
+	lowmodel "github.com/pb33f/libopenapi/datamodel/low"
+	lowbase "github.com/pb33f/libopenapi/datamodel/low/base"
 	low "github.com/pb33f/libopenapi/datamodel/low/v3"
 )
 
@@ -31,9 +35,21 @@ func NewConfigExplorer(spec high.Document, cfg config.Config) Explorer {
 }
 
 func (e configExplorer) FindProvider() (Provider, error) {
-	return Provider{
+	foundProvider := Provider{
 		Name: e.config.Provider.Name,
-	}, nil
+	}
+
+	if e.config.Provider.SchemaRef == "" {
+		return foundProvider, nil
+	}
+
+	schemaProxy, err := extractSchemaProxy(e.spec, e.config.Provider.SchemaRef)
+	if err != nil {
+		return Provider{}, fmt.Errorf("error extracting provider schema from ref: %w", err)
+	}
+	foundProvider.SchemaProxy = schemaProxy
+
+	return foundProvider, nil
 }
 
 func (e configExplorer) FindResources() (map[string]Resource, error) {
@@ -87,3 +103,30 @@ func extractOp(paths *high.Paths, oasLocation *config.OpenApiSpecLocation) *high
 		return nil
 	}
 }
+
+func extractSchemaProxy(document high.Document, componentRef string) (*highbase.SchemaProxy, error) {
+	// find the reference using the root document.Index
+	indexRef := document.Index.FindComponentInRoot(componentRef)
+	if indexRef == nil {
+		return nil, fmt.Errorf("unable to find reference: %s", componentRef)
+	}
+
+	// build low-level schema using YAML node
+	var lowSchema lowbase.Schema
+	err := lowmodel.BuildModel(indexRef.Node, &lowSchema)
+	if err != nil {
+		return nil, fmt.Errorf("error building low-level schema: %w", err)
+	}
+
+	// populate low-level schema, using root document.Index for resolving
+	err = lowSchema.Build(indexRef.Node, document.Index)
+	if err != nil {
+		return nil, fmt.Errorf("error populating low-level schema: %w", err)
+	}
+
+	// build high-level schema from low-level schema
+	highSchema := highbase.NewSchema(&lowSchema)
+
+	// wrap in a schema proxy for mapping with `oas` package
+	return highbase.CreateSchemaProxy(highSchema), nil
+}