Source enum values from external proto enums and reference external enum types

config/site/support/doctest_helper.rb

@@ -103,6 +103,21 @@ module ElasticGraph
       end
     end
 
+    doctest.before "ElasticGraph::ProtoIngestion::SchemaDefinition::SchemaElements::EnumTypeExtension#external_proto_enum" do
+      extend ::RSpec::Mocks::ExampleMethods
+
+      # The examples source enum values from an app-defined proto enum class; provide one here.
+      proto_enum_entry = ::Data.define(:name, :number)
+      currency_proto_enum = ::Class.new
+      currency_proto_enum.define_singleton_method(:enums) do
+        [:CURRENCY_UNKNOWN_DO_NOT_USE, :CURRENCY_USD, :CURRENCY_CAD].each_with_index.map do |name, number|
+          proto_enum_entry.new(name: name, number: number)
+        end
+      end
+
+      stub_const("MyApp::Protos::Currency", currency_proto_enum)
+    end
+
     doctest.before "ElasticGraph::SchemaDefinition::SchemaElements::ScalarType#coerce_with" do
       ::FileUtils.mkdir_p "coercion_adapters"
       ::File.write("coercion_adapters/phone_number.rb", <<~EOS)

elasticgraph-proto_ingestion/README.md

@@ -205,6 +205,84 @@ enums:
       BLUE: 2
 ```
 
+### Sourcing Enum Values From Existing Protobuf Enums
+
+If your project already has canonical protobuf enum definitions, you can source an enum's
+generated proto values from them instead of maintaining the value list in two places. Call
+`external_proto_enum` on the enum type with a proto enum class (anything exposing `.enums`).
+In the examples below, this stand-in plays the role of your app's generated proto enum class:
+
+```ruby
+# in config/schema/app_protos.rb
+
+# Stands in for a proto enum class generated by your app's protobuf tooling.
+module MyApp
+  module Protos
+    class Currency
+      EnumEntry = ::Data.define(:name, :number)
+
+      def self.enums
+        [
+          EnumEntry.new(name: :CURRENCY_UNKNOWN_DO_NOT_USE, number: 0),
+          EnumEntry.new(name: :CURRENCY_USD, number: 1),
+          EnumEntry.new(name: :CURRENCY_CAD, number: 2)
+        ]
+      end
+    end
+  end
+end
+```
+
+Optional per-source options curate and transform the sourced values:
+
+```ruby
+# in config/schema/currency.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.enum_type "Currency" do |t|
+    t.values "USD", "CAD"
+    t.external_proto_enum MyApp::Protos::Currency,
+      # Proto values to omit from the generated enum.
+      exclusions: [:UNKNOWN_DO_NOT_USE],
+      # Values expected in the generated enum that the proto enum lacks.
+      expected_extras: [:LEGACY],
+      # Optional transform applied to each proto value name.
+      name_transform: ->(name) { name.sub(/\ACURRENCY_/, "") }
+  end
+end
+```
+
+When an enum has one or more external sources, `elasticgraph-proto_ingestion` uses them
+as the source of the generated enum's values. When multiple sources are registered for the
+same enum, they must all resolve to the same value set.
+
+### Referencing Existing Protobuf Types
+
+For enums that exactly match a canonical proto enum, you can go further and reference the
+existing proto type instead of generating a duplicate local enum. Pass `proto:` and `import:`
+to `external_proto_enum`, and `schema.proto` will import the named file and use the external
+type name directly:
+
+```ruby
+# in config/schema/currency.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.enum_type "Currency" do |t|
+    t.values "USD", "CAD"
+    t.external_proto_enum MyApp::Protos::Currency,
+      proto: "myapp.types.Currency",
+      import: "myapp/types/currency.proto"
+  end
+end
+```
+
+A referenced enum must have exactly one option-free source whose values match the
+ElasticGraph enum's values; transformed, curated, or multi-source enums stay generated
+locally so value curation remains explicit. The source's enum entries must also expose
+`.number`, and those numbers must agree with any numbers previously pinned in
+`proto_field_numbers.yaml` — otherwise switching to the external type would silently
+reinterpret existing wire data.
+
 ## Type Mappings
 
 The generated `schema.proto` uses these built-in scalar mappings:

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb

@@ -157,6 +157,13 @@ def enum_value_numbers_for(enum_name, value_names)
           end
         end
 
+        # Returns previously pinned numbers for a protobuf enum.
+        #
+        # @api private
+        def pinned_enum_value_numbers(enum_name)
+          @proto_enum_value_numbers_by_enum[enum_name] || {}
+        end
+
         # Returns the label for a protobuf field under the configured syntax.
         #
         # @api private

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb

@@ -18,6 +18,58 @@ module SchemaDefinition
       module SchemaElements
         # Extends EnumType with proto field type conversion.
         module EnumTypeExtension
+          # Describes an external proto enum registered as the source of this enum's generated values.
+          ExternalProtoEnumSource = ::Data.define(:proto_enum, :exclusions, :expected_extras, :name_transform)
+
+          # Describes an existing proto enum type referenced instead of generating a local definition.
+          ExternalProtoEnumReference = ::Data.define(:proto_name, :import)
+
+          # Sources this enum's generated proto values from an existing proto enum class. Passing
+          # `proto:` and `import:` references that enum directly instead of generating a local enum.
+          #
+          # @return [void]
+          def external_proto_enum(proto_enum, exclusions: [], expected_extras: [], name_transform: nil, proto: nil, import: nil)
+            unless proto_enum.respond_to?(:enums)
+              raise Errors::SchemaError, "`external_proto_enum` on `#{name}` must be given a proto enum class with `.enums`, " \
+                "but got: #{proto_enum.inspect}."
+            end
+
+            if proto || import
+              unless proto.is_a?(String) && !proto.empty? && import.is_a?(String) && !import.empty?
+                raise Errors::SchemaError, "`external_proto_enum` on `#{name}` must be given both `proto` and `import` " \
+                  "as non-empty Strings to reference an external proto enum type."
+              end
+              unless exclusions.empty? && expected_extras.empty? && name_transform.nil?
+                raise Errors::SchemaError, "`external_proto_enum` on `#{name}` cannot combine `proto`/`import` with " \
+                  "`exclusions`, `expected_extras`, or `name_transform`; transformed or curated enums must stay generated locally."
+              end
+
+              @external_proto_reference = ExternalProtoEnumReference.new(proto_name: proto, import: import)
+              @proto_name = nil
+            end
+
+            external_proto_enum_sources << ExternalProtoEnumSource.new(
+              proto_enum: proto_enum,
+              exclusions: exclusions.map(&:to_s),
+              expected_extras: expected_extras.map(&:to_s),
+              name_transform: name_transform
+            )
+            nil
+          end
+
+          # External proto enums registered via {#external_proto_enum}.
+          #
+          # @return [Array<ExternalProtoEnumSource>]
+          def external_proto_enum_sources
+            @external_proto_enum_sources ||= []
+          end
+
+          # The external proto enum type referenced instead of generating a local enum, if configured.
+          #
+          # @dynamic external_proto_reference
+          # @return [ExternalProtoEnumReference, nil]
+          attr_reader :external_proto_reference
+
           # Defines an enum value and immediately validates its protobuf name.
           #
           # @return [void]
@@ -49,6 +101,11 @@ def value(value_name, &block)
           #
           # @return [String]
           def to_proto(schema)
+            if external_proto_reference
+              validate_external_proto_reference(schema)
+              return nil
+            end
+
             render_proto_enum(schema)
           end
 
@@ -70,7 +127,14 @@ def referenced_proto_types
           #
           # @return [String]
           def proto_name
-            @proto_name ||= Identifier.enum_name(name)
+            @proto_name ||= external_proto_reference&.proto_name || Identifier.enum_name(name)
+          end
+
+          # Returns the proto file imported for an externally referenced enum.
+          #
+          # @return [String, nil]
+          def protobuf_import
+            external_proto_reference&.import
           end
 
           # @private
@@ -83,20 +147,33 @@ def configure_derived_scalar_type(scalar_type)
           private
 
           def render_proto_enum(schema)
-            values = values_by_name.values
-            value_numbers = schema.enum_value_numbers_for(proto_name, values_by_name.keys)
+            source_value_names = proto_enum_value_names
+            proto_value_names_by_source_name = source_value_names.to_h do |source_name|
+              [source_name, proto_enum_value_name(source_name)]
+            end
+            duplicate_names = proto_value_names_by_source_name.values.tally.select { |_, count| count > 1 }
+            if duplicate_names.any?
+              raise Errors::SchemaError, "Enum `#{name}` maps to duplicate proto enum value names: #{duplicate_names.keys.sort.join(", ")}."
+            end
+            value_numbers = schema.enum_value_numbers_for(proto_name, source_value_names)
 
             zero_value_name = "#{enum_value_prefix}_UNSPECIFIED"
+            if (source_name = proto_value_names_by_source_name.key(zero_value_name))
+              raise Errors::SchemaError, "Enum `#{name}` value `#{source_name}` maps to proto enum value name " \
+                "`#{zero_value_name}`, which conflicts with the generated zero value `#{zero_value_name}`."
+            end
 
             lines = ProtoDocumentation.comment_lines_for(doc_comment)
             lines << "enum #{proto_name} {"
             lines << "  // The default value when no enum value has been explicitly set. Do not use this value."
             lines << "  // See https://protobuf.dev/programming-guides/proto3/#enum-default."
             lines << "  #{zero_value_name} = 0;"
-            values.each do |raw_value|
-              value = raw_value # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension
-              lines.concat(ProtoDocumentation.comment_lines_for(value.doc_comment, indent: "  "))
-              lines << "  #{value.proto_name(enum_value_prefix)} = #{value_numbers.fetch(value.name)};"
+            proto_value_names_by_source_name.each do |source_name, proto_value_name|
+              if (raw_value = values_by_name[source_name])
+                value = raw_value # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension
+                lines.concat(ProtoDocumentation.comment_lines_for(value.doc_comment, indent: "  "))
+              end
+              lines << "  #{proto_value_name} = #{value_numbers.fetch(source_name)};"
             end
             lines << "}"
             lines.join("\n")
@@ -105,6 +182,82 @@ def render_proto_enum(schema)
           def enum_value_prefix
             Support::Casing.to_upper_snake(name)
           end
+
+          def proto_enum_value_name(value_name)
+            if (raw_value = values_by_name[value_name])
+              value = raw_value # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension
+              value.proto_name(enum_value_prefix)
+            else
+              Identifier.enum_value_name("#{enum_value_prefix}_#{Support::Casing.to_upper_snake(value_name)}")
+            end
+          end
+
+          def proto_enum_value_names
+            return values_by_name.keys if external_proto_enum_sources.empty?
+
+            values_by_source = external_proto_enum_sources.map { |source| enum_value_names_from_source(source) }
+            canonical_values = values_by_source.first
+            canonical_set = canonical_values.uniq.sort
+            if values_by_source.drop(1).any? { |source_values| source_values.uniq.sort != canonical_set }
+              raise Errors::SchemaError, "External proto enums for `#{name}` produce inconsistent value sets. " \
+                "Ensure each `external_proto_enum` source (with exclusions/expected_extras/name_transform) resolves to the same values."
+            end
+            canonical_values
+          end
+
+          def enum_value_names_from_source(source)
+            name_transform = source.name_transform || :itself.to_proc
+            mapped_values = source.proto_enum.enums.map { |entry| name_transform.call(entry.name.to_s).to_s }
+            (mapped_values - source.exclusions + source.expected_extras).uniq
+          rescue => e
+            raise Errors::SchemaError, "Failed loading external proto enum values for `#{name}` from `#{source.proto_enum}`: #{e.message}"
+          end
+
+          def validate_external_proto_reference(schema)
+            unless external_proto_enum_sources.one?
+              raise Errors::SchemaError, "External proto enum `#{name}` must use exactly one `external_proto_enum` " \
+                "source; multi-source enums cannot be safely referenced externally."
+            end
+
+            numbers_by_value_name = enum_value_numbers_from_source(external_proto_enum_sources.first)
+            external_names = numbers_by_value_name.keys.sort
+            elasticgraph_names = values_by_name.keys.map(&:to_s).uniq.sort
+            if external_names != elasticgraph_names
+              raise Errors::SchemaError, "External proto enum `#{name}` values do not match the ElasticGraph enum values. " \
+                "External values: #{external_names.join(", ")}. ElasticGraph values: #{elasticgraph_names.join(", ")}."
+            end
+
+            schema.pinned_enum_value_numbers(Identifier.enum_name(name)).each do |value_name, pinned_number|
+              external_number = numbers_by_value_name[value_name]
+              if external_number && external_number != pinned_number
+                raise Errors::SchemaError, "External proto enum `#{name}` assigns `#{value_name}` the number " \
+                  "#{external_number}, but previously dumped artifacts pin it to #{pinned_number}; referencing it would " \
+                  "silently reinterpret existing wire data."
+              end
+
+              conflicting_name = numbers_by_value_name.find do |external_name, number|
+                external_name != value_name && number == pinned_number
+              end&.first
+              if conflicting_name
+                raise Errors::SchemaError, "External proto enum `#{name}` assigns `#{conflicting_name}` the number " \
+                  "#{pinned_number}, which previously dumped artifacts pin to `#{value_name}`; referencing it would " \
+                  "silently reinterpret existing wire data."
+              end
+            end
+          end
+
+          def enum_value_numbers_from_source(source)
+            entries = source.proto_enum.enums
+            unless entries.all? { |entry| entry.respond_to?(:number) }
+              raise Errors::SchemaError, "External proto enum `#{name}` cannot be referenced: its enum entries " \
+                "must expose `.number` so its values can be verified against previously pinned numbers."
+            end
+            entries.to_h { |entry| [entry.name.to_s, entry.number] }
+          rescue Errors::SchemaError
+            raise
+          rescue => e
+            raise Errors::SchemaError, "Failed loading external proto enum values for `#{name}` from `#{source.proto_enum}`: #{e.message}"
+          end
         end
       end
     end

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs

@@ -45,6 +45,7 @@ module ElasticGraph
           name_in_index: ::String
         ) -> ::Integer
         def enum_value_numbers_for: (::String, ::Array[::String]) -> ::Hash[::String, ::Integer]
+        def pinned_enum_value_numbers: (::String) -> ::Hash[::String, ::Integer]
         def field_label: (bool) -> ::String?
 
         private

elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs

@@ -3,11 +3,38 @@ module ElasticGraph
     module SchemaDefinition
       module SchemaElements
         module EnumTypeExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::EnumType
+          class ExternalProtoEnumSource
+            attr_reader proto_enum: untyped
+            attr_reader exclusions: ::Array[::String]
+            attr_reader expected_extras: ::Array[::String]
+            attr_reader name_transform: untyped
+
+            def self.new: (
+              proto_enum: untyped,
+              exclusions: ::Array[::String],
+              expected_extras: ::Array[::String],
+              name_transform: untyped
+            ) -> instance
+          end
+
+          class ExternalProtoEnumReference
+            attr_reader proto_name: ::String
+            attr_reader import: ::String
+
+            def self.new: (proto_name: ::String, import: ::String) -> instance
+          end
+
           @proto_name: ::String?
+          @external_proto_enum_sources: ::Array[ExternalProtoEnumSource]?
+          @external_proto_reference: ExternalProtoEnumReference?
 
+          def external_proto_enum: (untyped, ?exclusions: ::Array[::String | ::Symbol], ?expected_extras: ::Array[::String | ::Symbol], ?name_transform: untyped, ?proto: ::String?, ?import: ::String?) -> void
+          def external_proto_enum_sources: () -> ::Array[ExternalProtoEnumSource]
+          attr_reader external_proto_reference: ExternalProtoEnumReference?
           def value: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension) -> void } -> void
           def proto_name: () -> ::String
-          def to_proto: (Schema) -> ::String
+          def protobuf_import: () -> ::String?
+          def to_proto: (Schema) -> ::String?
           def proto_definition_kind: () -> :enum
           def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType]
           def configure_derived_scalar_type: (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType) -> void
@@ -16,6 +43,11 @@ module ElasticGraph
 
           def render_proto_enum: (Schema) -> ::String
           def enum_value_prefix: () -> ::String
+          def proto_enum_value_name: (::String) -> ::String
+          def proto_enum_value_names: () -> ::Array[::String]
+          def enum_value_names_from_source: (ExternalProtoEnumSource) -> ::Array[::String]
+          def validate_external_proto_reference: (Schema) -> void
+          def enum_value_numbers_from_source: (ExternalProtoEnumSource) -> ::Hash[::String, ::Integer]
         end
       end
     end