FDN-4939: Add OpenAPI 3.x import support

api/app/lib/OriginalUtil.scala

@@ -31,6 +31,8 @@ object OriginalUtil {
           Some(OriginalType.ApiJson)
         } else if (o.asOpt[Service].isDefined) {
           Some(OriginalType.ServiceJson)
+        } else if ((o \ "openapi").asOpt[JsString].exists(_.value.startsWith("3."))) {
+          Some(OriginalType.UNDEFINED("open_api_3"))
         } else if ((o \ "swagger").asOpt[JsString].isDefined) {
           Some(OriginalType.Swagger)
         } else {
@@ -40,6 +42,8 @@ object OriginalUtil {
       case _ => {
         if (trimmed.indexOf("protocol ") >= 0 || trimmed.indexOf("@namespace") >= 0) {
           Some(OriginalType.AvroIdl)
+        } else if (isOpenApi3Yaml(trimmed)) {
+          Some(OriginalType.UNDEFINED("open_api_3"))
         } else if (trimmed.contains("swagger:")) {
           Some(OriginalType.Swagger)
         } else {
@@ -49,6 +53,11 @@ object OriginalUtil {
     }
   }
 
+  private val OpenApi3YamlPattern = """(?m)^openapi:\s*["']?3\.""".r
+
+  private def isOpenApi3Yaml(data: String): Boolean =
+    OpenApi3YamlPattern.findFirstIn(data).isDefined
+
   private def guessApiOrServiceJson(o: JsObject): Option[OriginalType] = {
     // service.json has these defined as array; api.json as maps
     val fields = Seq("enums", "interfaces", "unions", "models")

build.sbt

@@ -61,12 +61,32 @@ lazy val swagger = project
     )
   )
 
+lazy val openapi = project
+  .in(file("openapi"))
+  .dependsOn(lib % "compile->compile;test->test")
+  .aggregate(lib)
+  .settings(
+    scalacOptions ++= allScalacOptions,
+    resolvers += "jitpack" at "https://jitpack.io",
+    libraryDependencies ++= Seq(
+      "com.softwaremill.sttp.apispec" %% "openapi-model" % "0.11.10",
+      "com.softwaremill.sttp.apispec" %% "openapi-circe" % "0.11.10",
+      "com.softwaremill.sttp.apispec" %% "openapi-circe-yaml" % "0.11.10",
+      "com.github.apicollective" % "apibuilder-validation" % "0.5.8",
+      "org.scalatestplus.play" %% "scalatestplus-play" % "7.0.1" % Test,
+    ),
+    Test / javaOptions ++= Seq(
+      "--add-exports=java.base/sun.security.x509=ALL-UNNAMED",
+      "--add-opens=java.base/sun.security.ssl=ALL-UNNAMED"
+    )
+  )
+
 val circeVersion = "0.14.9"
 lazy val core = project
   .in(file("core"))
   .enablePlugins(PlayScala)
-  .dependsOn(generated, lib, avro, swagger)
-  .aggregate(generated, lib, avro, swagger)
+  .dependsOn(generated, lib, avro, swagger, openapi)
+  .aggregate(generated, lib, avro, swagger, openapi)
   .settings(commonSettings*)
   .settings(
     libraryDependencies ++= Seq(

core/app/builder/OriginalValidator.scala

@@ -5,6 +5,7 @@ import cats.data.ValidatedNec
 import core.{ServiceFetcher, VersionMigration}
 import io.apibuilder.api.v0.models.OriginalType
 import io.apibuilder.avro.AvroIdlServiceValidator
+import io.apibuilder.openapi.OpenApiServiceValidator
 import io.apibuilder.spec.v0.models.Service
 import io.apibuilder.swagger.SwaggerServiceValidator
 import lib.{ServiceConfiguration, ServiceValidator}
@@ -23,6 +24,7 @@ object OriginalValidator {
       case OriginalType.ServiceJson => ServiceJsonServiceValidator
       case OriginalType.Swagger => SwaggerServiceValidator(config)
       case OriginalType.UNDEFINED("swagger_json") => SwaggerServiceValidator(config)
+      case OriginalType.UNDEFINED("open_api_3") => OpenApiServiceValidator(config)
       case OriginalType.UNDEFINED(other) => sys.error(s"Invalid original type[$other]")
     }
     WithServiceSpecValidator(validator)

openapi/src/main/scala/io/apibuilder/openapi/Classification.scala

@@ -0,0 +1,95 @@
+package io.apibuilder.openapi
+
+import io.apibuilder.spec.v0.models.Header
+import io.apibuilder.validation.ScalarType
+import sttp.apispec.Schema
+import sttp.apispec.openapi.OpenAPI
+
+import scala.collection.immutable.ListMap
+
+sealed trait SchemaKind
+object SchemaKind {
+  case object Object extends SchemaKind
+  case object StringEnum extends SchemaKind
+  case object Array extends SchemaKind
+  case object Union extends SchemaKind
+  case object Alias extends SchemaKind
+  case object Skip extends SchemaKind
+}
+
+sealed trait FieldKind
+object FieldKind {
+  case class Ref(typeName: String) extends FieldKind
+  case class AllOfRef(typeName: String) extends FieldKind
+  case object Number extends FieldKind
+  case class ArrayRef(typeName: String) extends FieldKind
+  case class ArrayEnum(itemsSchema: Schema) extends FieldKind
+  case class InlineEnum(enumSchema: Schema) extends FieldKind
+  case class ArraySimple(scalarType: ScalarType) extends FieldKind
+  case class MapType(valueType: String) extends FieldKind
+  case class Primitive(scalarType: ScalarType) extends FieldKind
+  case object DefaultedString extends FieldKind
+}
+
+case class ClassifiedField(
+  schemaName: String,
+  fieldName: String,
+  kind: Option[FieldKind],
+  description: Option[String],
+  required: Boolean,
+  minimum: Option[Long],
+  maximum: Option[Long],
+  annotations: SchemaClassifier.FieldAnnotations,
+)
+
+case class ClassifiedSchema(
+  name: String,
+  kind: SchemaKind,
+  schema: Option[Schema],
+  fields: Seq[ClassifiedField],
+)
+
+case class SchemaClassification(
+  schemas: Seq[ClassifiedSchema],
+)
+
+case class Classification(
+  classification: SchemaClassification,
+  modelReferences: Map[String, String],
+  pathResult: PathConversionResult,
+  securityHeaders: Seq[Header],
+  unsupportedFeatures: Seq[String],
+  openApi: OpenAPI,
+)
+
+object Classification {
+
+  def fromOpenApi(openApi: OpenAPI, namingConfig: NamingConfig, filterHeaders: Set[String]): Classification = {
+    val schemas = openApi.components
+      .map(_.schemas)
+      .getOrElse(ListMap.empty)
+
+    val modelReferences = SchemaResolver.buildModelReferences(schemas)
+    val classification = SchemaClassifier.classify(schemas)
+
+    val requestBodies = openApi.components
+      .map(_.requestBodies)
+      .getOrElse(ListMap.empty)
+
+    val securitySchemes = openApi.components
+      .map(_.securitySchemes)
+      .getOrElse(ListMap.empty)
+
+    val convertibleSchemeNames = securitySchemes.collect {
+      case (name, Right(scheme)) if SecurityConverter.isConvertible(scheme) => name
+    }.toSet
+
+    val pathConverter = new PathConverter(modelReferences, namingConfig, filterHeaders, requestBodies, convertibleSchemeNames)
+    val pathResult = pathConverter.convertPaths(openApi.paths)
+
+    val securityResult = SecurityConverter.convertSecuritySchemes(securitySchemes)
+    val unsupportedFeatures = ConversionReport.detectUnsupportedFeatures(openApi) ++ securityResult.degradedNotes
+
+    Classification(classification, modelReferences, pathResult, securityResult.headers, unsupportedFeatures, openApi)
+  }
+}

openapi/src/main/scala/io/apibuilder/openapi/ConversionReport.scala

@@ -0,0 +1,122 @@
+package io.apibuilder.openapi
+
+import sttp.apispec.openapi.OpenAPI
+
+case class SchemaReport(name: String, kind: SchemaKind)
+case class FieldReport(schemaName: String, fieldName: String, kind: Option[FieldKind], ignoredFormat: Option[String] = None)
+case class PathReport(path: String, methods: Seq[String], unsupported: Seq[String])
+
+case class ConversionReport(
+  schemas: Seq[SchemaReport],
+  fields: Seq[FieldReport],
+  paths: Seq[PathReport],
+  unsupportedFeatures: Seq[String],
+) {
+
+  def models: Seq[String] = schemas.collect { case SchemaReport(n, SchemaKind.Object) => n }
+  def enums: Seq[String] = schemas.collect { case SchemaReport(n, SchemaKind.StringEnum) => n }
+  def arrays: Seq[String] = schemas.collect { case SchemaReport(n, SchemaKind.Array) => n }
+  def unions: Seq[String] = schemas.collect { case SchemaReport(n, SchemaKind.Union) => n }
+  def aliases: Seq[String] = schemas.collect { case SchemaReport(n, SchemaKind.Alias) => n }
+  def skipped: Seq[String] = schemas.collect { case SchemaReport(n, SchemaKind.Skip) => n }
+
+  def unmappedFields: Seq[String] = fields.collect { case FieldReport(schema, field, None, _) => s"$schema.$field" }
+
+  def defaultedFields: Seq[String] = fields.collect {
+    case FieldReport(schema, field, Some(FieldKind.DefaultedString), _) => s"$schema.$field"
+  }
+
+  def ignoredFormats: Seq[String] = fields.collect { case FieldReport(schema, field, _, Some(fmt)) =>
+    s"$schema.$field (format: $fmt)"
+  }
+
+  def briefSummary: String = {
+    val pathIssueCount = paths.flatMap(_.unsupported).size
+    val parts = Seq(
+      Option.when(unmappedFields.nonEmpty)(s"${unmappedFields.size} unmapped fields"),
+      Option.when(defaultedFields.nonEmpty)(s"${defaultedFields.size} fields defaulted to string"),
+      Option.when(ignoredFormats.nonEmpty)(s"${ignoredFormats.size} ignored formats"),
+      Option.when(pathIssueCount > 0)(s"$pathIssueCount path issues"),
+      Option.when(unsupportedFeatures.nonEmpty)(s"${unsupportedFeatures.size} unsupported features"),
+    ).flatten
+    if (parts.isEmpty) "Imported from OpenAPI."
+    else s"Imported from OpenAPI. Conversion issues: ${parts.mkString(", ")}."
+  }
+
+  def summary: String = {
+    val lines = Seq.newBuilder[String]
+    lines += s"=== Conversion Report ==="
+    lines += s"Schemas: ${schemas.size} total"
+    lines += s"  Models:  ${models.size}"
+    lines += s"  Enums:   ${enums.size}"
+    lines += s"  Arrays:  ${arrays.size}"
+    lines += s"  Unions:  ${unions.size}"
+    lines += s"  Aliases: ${aliases.size} (resolved, not emitted)"
+    lines += s"  Skipped: ${skipped.size}"
+    if (unmappedFields.nonEmpty) {
+      lines += s"Unmapped fields: ${unmappedFields.size}"
+      unmappedFields.foreach(f => lines += s"  - $f")
+    }
+    if (defaultedFields.nonEmpty) {
+      lines += s"Defaulted to string (no type in spec): ${defaultedFields.size}"
+      defaultedFields.foreach(f => lines += s"  - $f")
+    }
+    if (ignoredFormats.nonEmpty) {
+      lines += s"Ignored formats: ${ignoredFormats.size}"
+      ignoredFormats.foreach(f => lines += s"  - $f")
+    }
+    lines += s"Paths: ${paths.size}"
+    val pathIssues = paths.flatMap(_.unsupported)
+    if (pathIssues.nonEmpty) {
+      lines += s"Path issues: ${pathIssues.size}"
+      pathIssues.foreach(i => lines += s"  - $i")
+    }
+    if (unsupportedFeatures.nonEmpty) {
+      lines += s"Unsupported features: ${unsupportedFeatures.size}"
+      unsupportedFeatures.foreach(f => lines += s"  - $f")
+    }
+    lines.result().mkString("\n")
+  }
+}
+
+object ConversionReport {
+
+  def fromClassification(c: Classification): ConversionReport = {
+    val schemaReports = c.classification.schemas.map(cs => SchemaReport(cs.name, cs.kind))
+    val fieldReports = c.classification.schemas.flatMap(_.fields.map { cf =>
+      FieldReport(cf.schemaName, cf.fieldName, cf.kind, cf.annotations.ignoredFormat)
+    })
+    ConversionReport(schemaReports, fieldReports, c.pathResult.pathReports, c.unsupportedFeatures)
+  }
+
+  def detectUnsupportedFeatures(openApi: OpenAPI): Seq[String] = {
+    val features = Seq.newBuilder[String]
+
+    openApi.components.foreach { c =>
+      c.securitySchemes.foreach {
+        case (name, Right(scheme)) if !SecurityConverter.isConvertible(scheme) =>
+          features += s"securityScheme '$name': type '${scheme.`type`}' (not converted)"
+        case (name, Left(_)) =>
+          features += s"securityScheme '$name': reference (not resolved)"
+        case _ => ()
+      }
+      if (c.requestBodies.nonEmpty)
+        features += s"requestBodies: ${c.requestBodies.size} defined (not converted as standalone)"
+      if (c.headers.nonEmpty)
+        features += s"headers: ${c.headers.size} defined (not converted)"
+      if (c.links.nonEmpty)
+        features += s"links: ${c.links.size} defined (not converted)"
+      if (c.callbacks.nonEmpty)
+        features += s"callbacks: ${c.callbacks.size} defined (not converted)"
+    }
+
+    if (openApi.security.nonEmpty)
+      features += s"global security: ${openApi.security.size} requirements (not converted)"
+    if (openApi.tags.nonEmpty)
+      features += s"tags: ${openApi.tags.size} defined (not converted)"
+    if (openApi.extensions.nonEmpty)
+      features += s"extensions: ${openApi.extensions.size} vendor extensions (not converted)"
+
+    features.result()
+  }
+}

openapi/src/main/scala/io/apibuilder/openapi/Converter.scala

@@ -0,0 +1,58 @@
+package io.apibuilder.openapi
+
+import io.apibuilder.spec.v0.models._
+import lib.{ServiceConfiguration, UrlKey}
+import play.api.libs.json.Json
+
+object Converter {
+
+  def convert(
+    openApi: sttp.apispec.openapi.OpenAPI,
+    config: ServiceConfiguration,
+    filterHeaders: Set[String] = Set.empty,
+    nameOverride: Option[String] = None,
+  ): Service = {
+    val namingConfig = NamingConfig()
+    val c = Classification.fromOpenApi(openApi, namingConfig, filterHeaders)
+    val schemaConverter = new SchemaConverter(c.modelReferences, namingConfig)
+    val schemaResult = schemaConverter.convert(c.classification)
+
+    val apiName = nameOverride.getOrElse(UrlKey.generate(openApi.info.title))
+    val report = ConversionReport.fromClassification(c)
+    val description = Seq(openApi.info.description, Some(report.briefSummary)).flatten.mkString("\n\n")
+    val conversionAttribute = buildConversionAttribute(report)
+
+    Service(
+      apidoc = None,
+      name = openApi.info.title,
+      organization = Organization(key = config.orgKey),
+      application = Application(key = apiName),
+      namespace = config.applicationNamespace(apiName),
+      version = config.version,
+      baseUrl = openApi.servers.headOption.map(_.url),
+      description = Some(description),
+      info = Info(license = None, contact = None),
+      headers = c.securityHeaders,
+      imports = Seq.empty,
+      enums = schemaResult.enums,
+      interfaces = Seq.empty,
+      unions = schemaResult.unions,
+      models = schemaResult.models,
+      resources = c.pathResult.resources,
+      attributes = Seq(conversionAttribute),
+      annotations = Seq.empty,
+    )
+  }
+
+  private def buildConversionAttribute(report: ConversionReport): Attribute = {
+    val pathIssues = report.paths.flatMap(_.unsupported)
+    val value = Json.obj(
+      "unmapped_fields"      -> report.unmappedFields,
+      "defaulted_fields"     -> report.defaultedFields,
+      "ignored_formats"      -> report.ignoredFormats,
+      "path_issues"          -> pathIssues,
+      "unsupported_features" -> report.unsupportedFeatures,
+    )
+    Attribute(name = "openapi_conversion", value = value)
+  }
+}