From 8d05d784fab50bf8a875717b1af358468f5c6dd8 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Fri, 17 Jul 2026 09:55:54 -0500 Subject: [PATCH 1/7] Generate a protobuf schema artifact from ElasticGraph schemas Fill in elasticgraph-proto_ingestion with proto schema generation for indexed types. Map built-in and custom scalars, render concrete and abstract types, preserve public field names, and generate enum defaults. Reject unrepresentable lists and unstable identifier collisions with actionable schema errors. --- elasticgraph-proto_ingestion/README.md | 103 +++++++ .../elasticgraph-proto_ingestion.gemspec | 3 + .../lib/elastic_graph/proto_ingestion.rb | 3 - .../schema_definition/api_extension.rb | 44 ++- .../schema_definition/factory_extension.rb | 149 +++++++++ .../schema_definition/identifier.rb | 86 ++++++ .../proto_ingestion_state.rb | 36 +++ .../schema_definition/results_extension.rb | 40 +++ .../schema_definition/schema.rb | 72 +++++ .../schema_artifact_manager_extension.rb | 41 +++ .../schema_elements/enum_type_extension.rb | 110 +++++++ .../schema_elements/enum_value_extension.rb | 29 ++ .../object_interface_and_union_extension.rb | 135 ++++++++ .../schema_elements/proto_documentation.rb | 33 ++ .../schema_elements/scalar_type_extension.rb | 74 +++++ .../schema_definition/state_extension.rb | 30 ++ .../sig/elastic_graph/proto_ingestion.rbs | 1 - .../schema_definition/api_extension.rbs | 8 +- .../schema_definition/factory_extension.rbs | 22 ++ .../schema_definition/identifier.rbs | 20 ++ .../proto_ingestion_state.rbs | 17 ++ .../schema_definition/results_extension.rbs | 15 + .../schema_definition/schema.rbs | 24 ++ .../schema_artifact_manager_extension.rbs | 12 + .../schema_elements/enum_type_extension.rbs | 23 ++ .../schema_elements/enum_value_extension.rbs | 13 + .../object_interface_and_union_extension.rbs | 32 ++ .../schema_elements/proto_documentation.rbs | 11 + .../schema_elements/scalar_type_extension.rbs | 20 ++ .../schema_definition/state_extension.rbs | 11 + .../schema_definition/rake_tasks_spec.rb | 87 ++++++ .../spec/spec_helper.rb | 10 + .../spec/support/proto_schema_support.rb | 61 ++++ .../schema_definition/api_extension_spec.rb | 60 +++- .../factory_extension_spec.rb | 60 ++++ .../schema_definition/identifier_spec.rb | 55 ++++ .../schema_artifact_manager_extension_spec.rb | 56 ++++ .../schema_edge_cases_spec.rb | 91 ++++++ .../schema_definition/schema_spec.rb | 288 ++++++++++++++++++ .../elastic_graph/proto_ingestion_spec.rb | 21 -- 40 files changed, 1966 insertions(+), 40 deletions(-) create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/identifier.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb create mode 100644 elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/factory_extension.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/identifier.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/results_extension.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs create mode 100644 elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/state_extension.rbs create mode 100644 elasticgraph-proto_ingestion/spec/integration/elastic_graph/proto_ingestion/schema_definition/rake_tasks_spec.rb create mode 100644 elasticgraph-proto_ingestion/spec/support/proto_schema_support.rb create mode 100644 elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/factory_extension_spec.rb create mode 100644 elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/identifier_spec.rb create mode 100644 elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb create mode 100644 elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb create mode 100644 elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb delete mode 100644 elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb diff --git a/elasticgraph-proto_ingestion/README.md b/elasticgraph-proto_ingestion/README.md index ca9ad4bcb..ba6ffe1b6 100644 --- a/elasticgraph-proto_ingestion/README.md +++ b/elasticgraph-proto_ingestion/README.md @@ -1,6 +1,7 @@ # ElasticGraph::ProtoIngestion An ElasticGraph extension that supports ingesting Protocol Buffer data into ElasticGraph. +Currently it generates `proto3` Protocol Buffers schema artifacts from ElasticGraph schemas. ## Dependency Diagram @@ -15,3 +16,105 @@ graph LR; elasticgraph-proto_ingestion --> elasticgraph-support; class elasticgraph-support otherEgGemStyle; ``` + +## Usage + +First, add `elasticgraph-proto_ingestion` to your `Gemfile`, alongside the other ElasticGraph gems: + +```diff +diff --git a/Gemfile b/Gemfile +index 4a5ef1e..5c16c2b 100644 +--- a/Gemfile ++++ b/Gemfile +@@ -8,6 +8,7 @@ gem "elasticgraph-query_registry", *elasticgraph_details + + # Can be elasticgraph-elasticsearch or elasticgraph-opensearch based on the datastore you want to use. + gem "elasticgraph-opensearch", *elasticgraph_details ++gem "elasticgraph-proto_ingestion", *elasticgraph_details + + gem "httpx", "~> 1.3" + +``` + +Next, update your `Rakefile` so that `ElasticGraph::ProtoIngestion::SchemaDefinition::APIExtension` is +included in the schema-definition extension modules: + +```diff +diff --git a/Rakefile b/Rakefile +index 2943335..26633c3 100644 +--- a/Rakefile ++++ b/Rakefile +@@ -3,5 +3,6 @@ + require "elastic_graph/json_ingestion/schema_definition/api_extension" + require "elastic_graph/local/rake_tasks" ++require "elastic_graph/proto_ingestion/schema_definition/api_extension" + require "elastic_graph/query_registry/rake_tasks" + require "rspec/core/rake_task" + require "standard/rake" +@@ -16,6 +17,7 @@ ElasticGraph::Local::RakeTasks.new( + # Determines casing of field names. Can be either `:camelCase` or `:snake_case`. + tasks.schema_element_name_form = :camelCase + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension ++ tasks.schema_definition_extension_modules << ElasticGraph::ProtoIngestion::SchemaDefinition::APIExtension + + # Customizes the names of fields generated by ElasticGraph. + tasks.schema_element_name_overrides = { +``` + +Adding the schema definition extension automatically enables `schema.proto` generation with the default +`elasticgraph` package. Optionally, configure a custom package name from your schema definition: + +```ruby +# in config/schema/protobuf.rb + +ElasticGraph.define_schema do |schema| + schema.proto_schema_artifacts package_name: "myapp.events.v1" +end +``` + +After running `bundle exec rake schema_artifacts:dump`, ElasticGraph will generate `schema.proto`. + +## Schema Definition API + +### Custom Scalar Types + +Built-in ElasticGraph scalar types are automatically mapped to proto scalar types. +For custom scalar types, use `protobuf` to define the proto scalar type: + +```ruby +# in config/schema/email.rb + +ElasticGraph.define_schema do |schema| + schema.scalar_type "Email" do |t| + t.mapping type: "keyword" + t.json_schema type: "string", format: "email" + t.protobuf type: "string" + end +end +``` + +## Type Mappings + +The generated `schema.proto` uses these built-in scalar mappings: + +| ElasticGraph Type | Protobuf Type | +|-------------------|------------| +| `Boolean` | `bool` | +| `Cursor` | `string` | +| `Date` | `string` | +| `DateTime` | `string` | +| `Float` | `double` | +| `ID` | `string` | +| `Int` | `int32` | +| `JsonSafeLong` | `int64` | +| `LocalTime` | `string` | +| `LongString` | `int64` | +| `String` | `string` | +| `TimeZone` | `string` | +| `Untyped` | `string` | + +Additionally: +- List types become `repeated` fields. +- Lists of lists (e.g. `[[Float!]!]!`) are not supported because Protocol Buffers cannot represent + them directly. Schema artifact generation raises an error identifying the unsupported field. +- Enum types generate `enum` definitions whose values are prefixed with the enum type name in `UPPER_SNAKE_CASE`, including a zero-valued `*_UNSPECIFIED` entry. diff --git a/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec b/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec index 7ef526536..e691205de 100644 --- a/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec +++ b/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec @@ -36,5 +36,8 @@ Gem::Specification.new do |spec| spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION + # This gem's schema-definition extension code references `elasticgraph-schema_definition`, but + # applications load it through schema-definition tasks after `elasticgraph-schema_definition` is already + # available. Keeping this as a development dependency avoids a runtime dependency cycle. spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb index df5f14429..e48ad46e2 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb @@ -11,8 +11,5 @@ module ElasticGraph module ProtoIngestion # The name of the generated Protocol Buffers schema file. PROTO_SCHEMA_FILE = "schema.proto" - - # The name of the generated proto field-number mapping file. - PROTO_FIELD_NUMBERS_FILE = "proto_field_numbers.yaml" end end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb index 761fea495..e0a987b47 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb @@ -6,7 +6,12 @@ # # frozen_string_literal: true +require "elastic_graph/errors" require "elastic_graph/proto_ingestion" +require "elastic_graph/proto_ingestion/schema_definition/factory_extension" +require "elastic_graph/proto_ingestion/schema_definition/identifier" +require "elastic_graph/proto_ingestion/schema_definition/schema" +require "elastic_graph/proto_ingestion/schema_definition/state_extension" module ElasticGraph module ProtoIngestion @@ -16,10 +21,43 @@ module ProtoIngestion module SchemaDefinition # Module designed to be extended onto an {ElasticGraph::SchemaDefinition::API} instance # to enable protobuf schema artifact generation. - # - # @note The protobuf schema artifact generation logic has not been implemented yet, so - # extending this module is currently a no-op. module APIExtension + # Wires up the protobuf extensions when this module is extended onto an API instance. + # + # @param api [ElasticGraph::SchemaDefinition::API] the API instance to extend + # @return [void] + # @api private + def self.extended(api) + api.state.extend(StateExtension) + api.factory.extend(FactoryExtension) + end + + # Configures protobuf artifact generation behavior. + # + # @param package_name [String] proto package name to emit + # @return [void] + # + # @example Set the proto package name + # ElasticGraph.define_schema do |schema| + # schema.proto_schema_artifacts package_name: "myapp.events.v1" + # end + def proto_schema_artifacts(package_name:) + if !package_name.is_a?(String) || package_name.empty? + raise Errors::SchemaError, "`package_name` must be a non-empty String" + end + + proto_ingestion_state.package_name = Identifier.validate_package_name(package_name) + nil + end + + private + + # Returns this gem's state container. Centralizes the Steep cast that's needed because + # Steep can't see the `extend(StateExtension)` applied at runtime in `extended`. + def proto_ingestion_state + extension_state = state # : ElasticGraph::SchemaDefinition::State & StateExtension + extension_state.proto_ingestion_state + end end end end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb new file mode 100644 index 000000000..e81f09a8f --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb @@ -0,0 +1,149 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/results_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Extension module applied to Factory to add proto support. + module FactoryExtension + # Default protobuf types applied to ElasticGraph's built-in scalar types as they are constructed. + BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME = { + "Boolean" => "bool", + "Cursor" => "string", + "Date" => "string", + "DateTime" => "string", + "Float" => "double", + "ID" => "string", + "Int" => "int32", + "JsonSafeLong" => "int64", + "LocalTime" => "string", + "LongString" => "int64", + "String" => "string", + "TimeZone" => "string", + "Untyped" => "string" + }.freeze + + # Creates a new enum type with proto extensions. + # + # @param name [String] enum type name + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::EnumType] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::EnumType] + def new_enum_type(name) + super(name) do |type| + extended_type = type.extend(SchemaElements::EnumTypeExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumType & SchemaElements::EnumTypeExtension + yield extended_type if block_given? + register_proto_type_name(extended_type) + end + end + + # Creates a new enum value with proto extensions. + # + # @param name [String] enum value name + # @param original_name [String] enum value name before overrides + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::EnumValue] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::EnumValue] + def new_enum_value(name, original_name) + super(name, original_name) do |value| + extended_value = value.extend(SchemaElements::EnumValueExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & SchemaElements::EnumValueExtension + yield extended_value if block_given? + end + end + + # Creates a new interface type with proto extensions. + # + # @param name [String] interface type name + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType] + def new_interface_type(name) + super(name) do |type| + extended_type = type.extend(SchemaElements::ObjectInterfaceAndUnionExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType & SchemaElements::ObjectInterfaceAndUnionExtension + yield extended_type if block_given? + register_proto_type_name(extended_type) + end + end + + # Creates a new object type with proto extensions. + # + # @param name [String] object type name + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::ObjectType] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::ObjectType] + def new_object_type(name) + super(name) do |type| + extended_type = type.extend(SchemaElements::ObjectInterfaceAndUnionExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::ObjectType & SchemaElements::ObjectInterfaceAndUnionExtension + yield extended_type if block_given? + register_proto_type_name(extended_type) + end + end + + # Creates a new scalar type with proto extensions. + # + # @param name [String] scalar type name + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::ScalarType] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::ScalarType] + def new_scalar_type(name) + super(name) do |type| + extended_type = type.extend(SchemaElements::ScalarTypeExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension + + if state.initially_registered_built_in_types.empty? && + (proto_type = BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME[name.to_s]) + extended_type.protobuf type: proto_type + end + + yield extended_type if block_given? + extended_type.finalize_protobuf_configuration! + end + end + + # Creates a new union type with proto extensions. + # + # @param name [String] union type name + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::UnionType] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::UnionType] + def new_union_type(name) + super(name) do |type| + extended_type = type.extend(SchemaElements::ObjectInterfaceAndUnionExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::UnionType & SchemaElements::ObjectInterfaceAndUnionExtension + yield extended_type if block_given? + register_proto_type_name(extended_type) + end + end + + # Creates a new results object and extends it with proto generation APIs. + # + # @return [ElasticGraph::SchemaDefinition::Results] + def new_results + super.tap do |results| + results.extend ResultsExtension + end + end + + # Creates a new schema artifact manager and extends it with proto artifact support. + # + # @return [ElasticGraph::SchemaDefinition::SchemaArtifactManager] + def new_schema_artifact_manager(...) + super.tap do |manager| + manager.extend SchemaArtifactManagerExtension + end + end + + private + + def register_proto_type_name(type) + extension_state = state # : ElasticGraph::SchemaDefinition::State & StateExtension + extension_state.proto_ingestion_state.register_proto_type_name(type.proto_name, type.name) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/identifier.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/identifier.rb new file mode 100644 index 000000000..cb2647448 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/identifier.rb @@ -0,0 +1,86 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Helpers for rendering Protocol Buffers identifiers while avoiding keyword conflicts. + class Identifier + # Matches a single valid protobuf identifier (e.g. one segment of a package name). + # https://protobuf.com/docs/language-spec#identifiers-and-keywords + VALID_IDENTIFIER = /\A[A-Za-z_][A-Za-z0-9_]*\z/ + + # @dynamic self.enum_name, self.field_name, self.enum_value_name + + class << self + # Validates a protobuf package identifier. + # + # @param name [#to_s] + # @return [String] + def validate_package_name(name) + name = name.to_s + segments = name.split(".", -1) + + if segments.empty? || segments.any? { |segment| !VALID_IDENTIFIER.match?(segment) } + raise Errors::SchemaError, "`package_name` must be a dot-separated list of protobuf identifiers " \ + "(each starting with a letter or underscore, containing only letters, digits, and underscores), " \ + "got: #{name.inspect}." + end + + name + end + + # Builds a protobuf message identifier. + # + # @param name [#to_s] + # @return [String] + def message_name(name) + escape_keyword(name.to_s) + end + + # Builds a protobuf enum identifier. + # + # @return [String] + alias_method :enum_name, :message_name + + # Builds a protobuf field identifier. + # + # @return [String] + alias_method :field_name, :message_name + + # Builds a protobuf enum value identifier. + # + # @return [String] + alias_method :enum_value_name, :message_name + + # Escapes protobuf reserved keywords by suffixing them with an underscore. + # + # @param identifier [String] + # @return [String] + private + + def escape_keyword(identifier) + return identifier unless PROTO_KEYWORDS.include?(identifier) + "#{identifier}_" + end + end + + # Reserved words in protobuf syntax that cannot be used as identifiers verbatim. + # + # @return [Set] + PROTO_KEYWORDS = ::Set[ + "bool", "bytes", "double", "enum", "false", "fixed32", "fixed64", "float", "import", "int32", "int64", "map", + "message", "oneof", "option", "package", "public", "repeated", "reserved", "rpc", "service", "sfixed32", "sfixed64", + "sint32", "sint64", "stream", "string", "syntax", "to", "true", "uint32", "uint64", "weak" + ].freeze + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb new file mode 100644 index 000000000..5dd914cda --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb @@ -0,0 +1,36 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Holds the proto ingestion extension's schema definition state. + # + # @private + class ProtoIngestionState < ::Struct.new(:package_name) + def initialize(...) + super + @type_name_by_proto_name = {} + end + + # Registers the protobuf name for a schema type, raising immediately if it collides with a prior type. + # + # @return [void] + def register_proto_type_name(proto_name, type_name) + existing_type_name = @type_name_by_proto_name.fetch(proto_name, type_name) + @type_name_by_proto_name[proto_name] = existing_type_name + return if existing_type_name == type_name + + raise Errors::SchemaError, "Type names `#{existing_type_name}` and `#{type_name}` both map to the same proto type name `#{proto_name}`." + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb new file mode 100644 index 000000000..50fa81213 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb @@ -0,0 +1,40 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/schema" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Extension module for {ElasticGraph::SchemaDefinition::Results} that adds proto schema generation support. + module ResultsExtension + # Returns the generated proto schema. + # + # @return [String] complete `proto3` schema file contents + def proto_schema + @proto_schema ||= protobuf_schema_generator.to_proto + end + + private + + def protobuf_schema_generator + # The cast is needed because Steep can't see the `extend(StateExtension)` applied at + # runtime in {APIExtension.extended}. + extension_state = state # : ElasticGraph::SchemaDefinition::State & StateExtension + ingestion_state = extension_state.proto_ingestion_state + + Schema.new( + state: extension_state, + all_types: all_types, + package_name: ingestion_state.package_name + ) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb new file mode 100644 index 000000000..11647e1d2 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb @@ -0,0 +1,72 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/identifier" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Builds a `proto3` schema string from an ElasticGraph schema definition. + class Schema + # @param state [ElasticGraph::SchemaDefinition::State] + # @param all_types [Array] + # @param package_name [String] + def initialize(state:, all_types:, package_name:) + @state = state + @all_types = all_types + @package_name = Identifier.validate_package_name(package_name) + end + + # Renders the schema as a valid `proto3` file. + # + # @return [String] + def to_proto + types = proto_types + return "" if types.empty? + + sections = [ + %(syntax = "proto3";), + "package #{@package_name};", + render_definitions(types) + ] + + sections.join("\n\n") + "\n" + end + + private + + # Selects the indexed root types and every type transitively referenced by their protobuf + # representations. All traversal state is local so repeated calls are independent. + def proto_types + types_to_visit = _ = @state.indexed_types_by_index_name.values.dup + type_names_to_render = ::Set.new + + while (type = types_to_visit.shift) + next unless type_names_to_render.add?(type.name) + + types_to_visit.concat(type.referenced_proto_types) + end + + @all_types.select do |type| + type_names_to_render.include?(type.name) && type.respond_to?(:to_proto) + end + end + + def render_definitions(types) + types + .sort_by { |type| [(type.proto_definition_kind == :enum) ? 0 : 1, type.proto_name] } + .filter_map(&:to_proto) + .join("\n\n") + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rb new file mode 100644 index 000000000..352c2df84 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rb @@ -0,0 +1,41 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Extension module for {ElasticGraph::SchemaDefinition::SchemaArtifactManager} that adds + # proto artifact generation support. + # + # @private + module SchemaArtifactManagerExtension + private + + # Overrides the base `artifacts_from_schema_def` method to add proto artifacts. + def artifacts_from_schema_def + base_artifacts = super + proto_schema = protobuf_schema_definition_results.proto_schema + return base_artifacts if proto_schema.empty? + + base_artifacts + [ + new_raw_artifact(PROTO_SCHEMA_FILE, proto_schema.chomp, comment_prefix: "//") + ] + end + + # Returns the wrapped {ElasticGraph::SchemaDefinition::Results} narrowed to include this + # gem's `ResultsExtension`. Centralizes the Steep cast that's needed because Steep can't + # see the `extend(ResultsExtension)` applied at runtime. + def protobuf_schema_definition_results + schema_definition_results # : ElasticGraph::SchemaDefinition::Results & ResultsExtension + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb new file mode 100644 index 000000000..37c2fbfe4 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb @@ -0,0 +1,110 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/proto_ingestion/schema_definition/identifier" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation" +require "elastic_graph/support/casing" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Protobuf schema definition extensions for ElasticGraph schema elements. + module SchemaElements + # Extends EnumType with proto field type conversion. + module EnumTypeExtension + # Defines an enum value and immediately validates its protobuf name. + # + # @return [void] + def value(value_name, &block) + super + new_value = values_by_name.values.last # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension + new_proto_name = new_value.proto_name(enum_value_prefix) + zero_value_name = "#{enum_value_prefix}_UNSPECIFIED" + + if new_proto_name == zero_value_name + raise Errors::SchemaError, "Enum `#{name}` value `#{new_value.name}` maps to proto enum value name " \ + "`#{new_proto_name}`, which conflicts with the generated zero value `#{zero_value_name}`." + end + + duplicate = values_by_name.values.find do |raw_value| + existing_value = raw_value # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension + !existing_value.equal?(new_value) && existing_value.proto_name(enum_value_prefix) == new_proto_name + end + + if duplicate + raise Errors::SchemaError, "Enum `#{name}` values `#{duplicate.name}` and `#{new_value.name}` " \ + "map to duplicate proto enum value name `#{new_proto_name}`." + end + + nil + end + + # Renders this enum's protobuf definition. + # + # @return [String] + def to_proto + render_proto_enum + end + + # Returns the kind used to order this definition in a protobuf schema. + # + # @return [Symbol] + def proto_definition_kind + :enum + end + + # Returns the schema types referenced by this definition. + # + # @return [Array] + def referenced_proto_types + [] + end + + # Returns this enum type's name in protobuf schemas. + # + # @return [String] + def proto_name + @proto_name ||= Identifier.enum_name(name) + end + + # @private + def configure_derived_scalar_type(scalar_type) + super + proto_scalar_type = scalar_type # : ::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & ScalarTypeExtension + proto_scalar_type.protobuf type: proto_name + end + + private + + def render_proto_enum + values = values_by_name.values + zero_value_name = "#{enum_value_prefix}_UNSPECIFIED" + + lines = ProtoDocumentation.comment_lines_for(doc_comment) + lines << "enum #{proto_name} {" + lines << " // The default value when no enum value has been explicitly set. Do not use this value." + lines << " // See https://protobuf.dev/programming-guides/proto3/#enum-default." + lines << " #{zero_value_name} = 0;" + values.each.with_index(1) do |raw_value, number| + value = raw_value # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension + lines.concat(ProtoDocumentation.comment_lines_for(value.doc_comment, indent: " ")) + lines << " #{value.proto_name(enum_value_prefix)} = #{number};" + end + lines << "}" + lines.join("\n") + end + + def enum_value_prefix + Support::Casing.to_upper_snake(name) + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rb new file mode 100644 index 000000000..3c597d914 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rb @@ -0,0 +1,29 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/identifier" +require "elastic_graph/support/casing" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + # Extends enum values with protobuf naming behavior. + module EnumValueExtension + # Returns this enum value's name in protobuf schemas. + # + # @param enum_value_prefix [String] normalized prefix of the containing enum + # @return [String] + def proto_name(enum_value_prefix) + @proto_name ||= Identifier.enum_value_name("#{enum_value_prefix}_#{Support::Casing.to_upper_snake(name)}") + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb new file mode 100644 index 000000000..9e6ded865 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb @@ -0,0 +1,135 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/proto_ingestion/schema_definition/identifier" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation" +require "elastic_graph/support/casing" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + # Extends object/interface/union types with proto field type conversion. + module ObjectInterfaceAndUnionExtension + # Renders this type's protobuf message definition. + # + # @return [String] + def to_proto + render_proto_message(proto_name) + end + + # Returns the kind used to order this definition in a protobuf schema. + # + # @return [Symbol] + def proto_definition_kind + :message + end + + # Returns the schema types referenced by this definition. + # + # @return [Array] + def referenced_proto_types + if abstract? + abstract_type = _ = self + abstract_type.recursively_resolve_subtypes + else + proto_fields.map do |_, field| + _ = field.type.fully_unwrapped.resolved + end + end + end + + # Returns this type's name in protobuf schemas. + # + # @return [String] + def proto_name + @proto_name ||= Identifier.message_name(name) + end + + private + + def render_proto_message(message_name) + return render_proto_oneof(message_name) if abstract? + + fields = proto_fields + field_names = fields.map { |_, field| Identifier.field_name(field.name) } + duplicate_names = field_names.tally.select { |_, count| count > 1 } + if duplicate_names.any? + raise Errors::SchemaError, "Type `#{name}` maps to duplicate proto field names: #{duplicate_names.keys.sort.join(", ")}." + end + + lines = ProtoDocumentation.comment_lines_for(doc_comment) + lines << "message #{message_name} {" + fields.each.with_index(1) do |(schema_field, field), field_number| + field_name = Identifier.field_name(field.name) + repeated, field_type = proto_field_type_for(field.type, context_field_name: field.name) + label = "repeated " if repeated + line = " #{label}#{field_type} #{field_name} = #{field_number};" + line += " // source name: #{field.name}" if field_name != field.name + lines.concat(ProtoDocumentation.comment_lines_for(schema_field.doc_comment, indent: " ")) + lines << line + end + lines << "}" + lines.join("\n") + end + + def render_proto_oneof(message_name) + # @type var abstract_type: ::ElasticGraph::SchemaDefinition::Mixins::HasSubtypes + abstract_type = _ = self + lines = ProtoDocumentation.comment_lines_for(doc_comment) + lines << "message #{message_name} {" + lines << " oneof value {" + abstract_type.recursively_resolve_subtypes.each.with_index(1) do |subtype, field_number| + subtype_name = (_ = subtype).proto_name + field_name = Identifier.field_name(Support::Casing.to_upper_snake(subtype_name).downcase) + lines << " #{subtype_name} #{field_name} = #{field_number};" + end + lines << " }" + lines << "}" + lines.join("\n") + end + + def proto_fields + indexing_fields_by_name_in_index.values.filter_map do |schema_field| + next if schema_field.name == "__typename" + + indexing_field = schema_field.to_indexing_field # : ElasticGraph::SchemaDefinition::Indexing::Field + [schema_field, indexing_field] # : [::ElasticGraph::SchemaDefinition::SchemaElements::Field, ::ElasticGraph::SchemaDefinition::Indexing::Field] + end + end + + def proto_field_type_for(type_ref, context_field_name:) + list_depth, base_type_ref = list_depth_and_base_type(type_ref) + + if list_depth > 1 + raise Errors::SchemaError, "Field `#{name}.#{context_field_name}` has type `#{type_ref.name}`, " \ + "but Protocol Buffers cannot represent lists of lists directly. " \ + "`elasticgraph-proto_ingestion` supports fields with at most one list level." + end + + proto_type = _ = base_type_ref.resolved + [list_depth == 1, proto_type.proto_name] + end + + def list_depth_and_base_type(type_ref) + list_depth = 0 + current = type_ref.unwrap_non_null + + while current.list? + list_depth += 1 + current = current.unwrap_list.unwrap_non_null + end + + [list_depth, current] + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rb new file mode 100644 index 000000000..0f4b8f32b --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rb @@ -0,0 +1,33 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + # Formats ElasticGraph schema element documentation as protobuf comments. + # + # @api private + module ProtoDocumentation + # Formats documentation as line comments. + # + # @param doc_comment [String, nil] schema element documentation + # @param indent [String] indentation to put before each comment + # @return [Array] formatted protobuf comment lines + def self.comment_lines_for(doc_comment, indent: "") + return [] unless doc_comment + + doc_comment.chomp.lines(chomp: true).map do |line| + line.empty? ? "#{indent}//" : "#{indent}// #{line}" + end + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb new file mode 100644 index 000000000..5a8c724a3 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb @@ -0,0 +1,74 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + # Extends ScalarType with proto field type conversion. + module ScalarTypeExtension + # Configured protobuf type (e.g. string, int64, bool). + # @dynamic protobuf_type + attr_reader :protobuf_type + + # Configures the protobuf type for this scalar type. + # + # @param type [String] protobuf scalar type name + # @return [void] + def protobuf(type:) + @protobuf_type = type + @proto_name = nil + end + + # Validates that a protobuf type has been configured on this scalar type. GraphQL-only + # scalar types are skipped because they are not part of ingestion. + # + # @return [void] + # @raise [Errors::SchemaError] when missing + def finalize_protobuf_configuration! + return if graphql_only? + proto_name + nil + end + + # Scalars map to protobuf field types and do not render standalone definitions. + # + # @return [nil] + def to_proto + nil + end + + # Returns the kind used to order this definition in a protobuf schema. + # + # @return [nil] + def proto_definition_kind + nil + end + + # Returns the schema types referenced by this definition. + # + # @return [Array] + def referenced_proto_types + [] + end + + # Returns this scalar's name in protobuf schemas. + # + # @return [String] + def proto_name + @proto_name ||= protobuf_type || + raise(Errors::SchemaError, "Protobuf type not configured for scalar type `#{name}`. " \ + 'To proceed, call `protobuf type: "TYPE"` on the scalar type definition.') + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb new file mode 100644 index 000000000..f59e96343 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb @@ -0,0 +1,30 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Extension module applied to `ElasticGraph::SchemaDefinition::State` to hold protobuf configuration. + # + # @private + module StateExtension + # @dynamic proto_ingestion_state + attr_reader :proto_ingestion_state + + def self.extended(state) + state.instance_variable_set( + :@proto_ingestion_state, + ProtoIngestionState.new(package_name: "elasticgraph") + ) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs index 81cc74611..7cb944875 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs @@ -1,6 +1,5 @@ module ElasticGraph module ProtoIngestion PROTO_SCHEMA_FILE: ::String - PROTO_FIELD_NUMBERS_FILE: ::String end end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs index 81e79cec4..e7bb15e78 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs @@ -1,7 +1,13 @@ module ElasticGraph module ProtoIngestion module SchemaDefinition - module APIExtension + module APIExtension: ::ElasticGraph::SchemaDefinition::API + def self.extended: (::ElasticGraph::SchemaDefinition::API & APIExtension) -> void + def proto_schema_artifacts: (package_name: ::String) -> void + + private + + def proto_ingestion_state: () -> ProtoIngestionState end end end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/factory_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/factory_extension.rbs new file mode 100644 index 000000000..4822f3fb3 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/factory_extension.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module FactoryExtension: ::ElasticGraph::SchemaDefinition::Factory + BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME: ::Hash[::String, ::String] + + def new_enum_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumType & SchemaElements::EnumTypeExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::EnumType & SchemaElements::EnumTypeExtension) + def new_enum_value: (::String, ::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & SchemaElements::EnumValueExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & SchemaElements::EnumValueExtension) + def new_interface_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType & SchemaElements::ObjectInterfaceAndUnionExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType & SchemaElements::ObjectInterfaceAndUnionExtension) + def new_object_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::ObjectType & SchemaElements::ObjectInterfaceAndUnionExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::ObjectType & SchemaElements::ObjectInterfaceAndUnionExtension) + def new_scalar_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension) + def new_union_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::UnionType & SchemaElements::ObjectInterfaceAndUnionExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::UnionType & SchemaElements::ObjectInterfaceAndUnionExtension) + def new_results: () -> ::ElasticGraph::SchemaDefinition::Results + def new_schema_artifact_manager: (**untyped) -> ::ElasticGraph::SchemaDefinition::SchemaArtifactManager + + private + + def register_proto_type_name: (untyped) -> void + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/identifier.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/identifier.rbs new file mode 100644 index 000000000..25c69322d --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/identifier.rbs @@ -0,0 +1,20 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + class Identifier + PROTO_KEYWORDS: ::Set[::String] + VALID_IDENTIFIER: ::Regexp + + def self.validate_package_name: (::String) -> ::String + def self.message_name: (::String) -> ::String + def self.enum_name: (::String) -> ::String + def self.field_name: (::String) -> ::String + def self.enum_value_name: (::String) -> ::String + + private + + def self.escape_keyword: (::String) -> ::String + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs new file mode 100644 index 000000000..964206a09 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs @@ -0,0 +1,17 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + class ProtoIngestionStateSupertype + attr_accessor package_name: ::String + + def initialize: (package_name: ::String) -> void + end + + class ProtoIngestionState < ProtoIngestionStateSupertype + def register_proto_type_name: (::String proto_name, ::String type_name) -> void + + @type_name_by_proto_name: ::Hash[::String, ::String] + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/results_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/results_extension.rbs new file mode 100644 index 000000000..96de3ae03 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/results_extension.rbs @@ -0,0 +1,15 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module ResultsExtension : ::ElasticGraph::SchemaDefinition::Results + def proto_schema: () -> ::String + + private + + def protobuf_schema_generator: () -> Schema + + @proto_schema: ::String? + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs new file mode 100644 index 000000000..7ff135db9 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs @@ -0,0 +1,24 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + class Schema + @state: ::ElasticGraph::SchemaDefinition::State + @all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] + @package_name: ::String + + def initialize: ( + state: ::ElasticGraph::SchemaDefinition::State, + all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType], + package_name: ::String + ) -> void + + def to_proto: () -> ::String + + private + + def proto_types: () -> ::Array[untyped] + def render_definitions: (::Array[untyped] types) -> ::String + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rbs new file mode 100644 index 000000000..4298c1a99 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rbs @@ -0,0 +1,12 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaArtifactManagerExtension : ::ElasticGraph::SchemaDefinition::SchemaArtifactManager + private + + def artifacts_from_schema_def: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]] + def protobuf_schema_definition_results: () -> (::ElasticGraph::SchemaDefinition::Results & ResultsExtension) + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs new file mode 100644 index 000000000..5d7c62c61 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs @@ -0,0 +1,23 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + module EnumTypeExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::EnumType + @proto_name: ::String? + + def value: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension) -> void } -> void + def proto_name: () -> ::String + def to_proto: () -> ::String + def proto_definition_kind: () -> :enum + def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] + def configure_derived_scalar_type: (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType) -> void + + private + + def render_proto_enum: () -> ::String + def enum_value_prefix: () -> ::String + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rbs new file mode 100644 index 000000000..c1d2fa996 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + module EnumValueExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue + @proto_name: ::String? + + def proto_name: (::String enum_value_prefix) -> ::String + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs new file mode 100644 index 000000000..3b2bf3f3f --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs @@ -0,0 +1,32 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + module ObjectInterfaceAndUnionExtension : ::ElasticGraph::SchemaDefinition::SchemaElements::ObjectType + @proto_name: ::String? + + def proto_name: () -> ::String + def to_proto: () -> ::String + def proto_definition_kind: () -> :message + def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] + + private + + def render_proto_message: (::String message_name) -> ::String + def render_proto_oneof: (::String message_name) -> ::String + def proto_fields: () -> ::Array[[ + ::ElasticGraph::SchemaDefinition::SchemaElements::Field, + ::ElasticGraph::SchemaDefinition::Indexing::Field + ]] + def proto_field_type_for: ( + ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference, + context_field_name: ::String + ) -> [bool, ::String] + def list_depth_and_base_type: ( + ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference + ) -> [::Integer, ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference] + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rbs new file mode 100644 index 000000000..c2cf49bd8 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + module ProtoDocumentation + def self.comment_lines_for: (::String?, ?indent: ::String) -> ::Array[::String] + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs new file mode 100644 index 000000000..817d6fee7 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs @@ -0,0 +1,20 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + module ScalarTypeExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType + @proto_name: ::String? + + attr_reader protobuf_type: ::String? + + def protobuf: (type: ::String) -> void + def finalize_protobuf_configuration!: () -> void + def proto_name: () -> ::String + def to_proto: () -> nil + def proto_definition_kind: () -> nil + def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/state_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/state_extension.rbs new file mode 100644 index 000000000..3b29bc557 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/state_extension.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module StateExtension: ::ElasticGraph::SchemaDefinition::State + attr_reader proto_ingestion_state: ProtoIngestionState + + def self.extended: (::ElasticGraph::SchemaDefinition::State & StateExtension) -> void + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/integration/elastic_graph/proto_ingestion/schema_definition/rake_tasks_spec.rb b/elasticgraph-proto_ingestion/spec/integration/elastic_graph/proto_ingestion/schema_definition/rake_tasks_spec.rb new file mode 100644 index 000000000..66fae144e --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/integration/elastic_graph/proto_ingestion/schema_definition/rake_tasks_spec.rb @@ -0,0 +1,87 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion" +require "elastic_graph/proto_ingestion/schema_definition/api_extension" +require "elastic_graph/schema_definition/rake_tasks" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe "Protobuf RakeTasks", :rake_task, :in_temp_dir do + describe "schema_artifacts:dump" do + it "dumps proto artifact when indexed types are defined" do + write_proto_schema(table_defs: <<~EOS) + s.object_type "Product" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "products" + end + EOS + + expect { + output = run_rake_with_proto("schema_artifacts:dump") + expect(output.lines).to include(a_string_including("Dumped", PROTO_SCHEMA_FILE)) + }.to change { read_artifact(PROTO_SCHEMA_FILE) } + .from(nil) + .to( + start_with( + "// Generated by `bundle exec rake schema_artifacts:dump`.\n" \ + "// DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run.\n" + ).and(a_string_including('syntax = "proto3";', "message Product", "string name = 2;")) + ) + end + + it "idempotently dumps proto artifacts" do + write_proto_schema(table_defs: <<~EOS) + s.object_type "Product" do |t| + t.field "id", "ID" + t.index "products" + end + EOS + + run_rake_with_proto("schema_artifacts:dump") + + expect { + output = run_rake_with_proto("schema_artifacts:dump") + expect(output.lines).to include(a_string_including("already up to date", PROTO_SCHEMA_FILE)) + }.to maintain { read_artifact(PROTO_SCHEMA_FILE) } + end + end + + private + + def write_proto_schema(table_defs:) + ::File.write("schema.rb", <<~EOS) + ElasticGraph.define_schema do |s| + #{table_defs} + end + EOS + end + + def run_rake_with_proto(*args) + run_rake(*args) do |output| + ElasticGraph::SchemaDefinition::RakeTasks.new( + schema_element_name_form: :snake_case, + index_document_sizes: false, + path_to_schema: "schema.rb", + schema_artifacts_directory: "config/schema/artifacts", + extension_modules: [SchemaDefinition::APIExtension], + output: output + ) + end + end + + def read_artifact(name) + path = File.join("config", "schema", "artifacts", name) + File.read(path) if File.exist?(path) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/spec_helper.rb b/elasticgraph-proto_ingestion/spec/spec_helper.rb index 4e6c40a9a..39286580a 100644 --- a/elasticgraph-proto_ingestion/spec/spec_helper.rb +++ b/elasticgraph-proto_ingestion/spec/spec_helper.rb @@ -8,3 +8,13 @@ # This file contains RSpec configuration for `elasticgraph-proto_ingestion`. # It is loaded by the shared spec helper at `spec_support/spec_helper.rb`. + +RSpec.configure do |config| + config.define_derived_metadata(absolute_file_path: %r{/elasticgraph-proto_ingestion/}) do |meta| + meta[:proto_schema] = true + end + + config.when_first_matching_example_defined(:proto_schema) do + require "support/proto_schema_support" + end +end diff --git a/elasticgraph-proto_ingestion/spec/support/proto_schema_support.rb b/elasticgraph-proto_ingestion/spec/support/proto_schema_support.rb new file mode 100644 index 000000000..20aa71487 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/support/proto_schema_support.rb @@ -0,0 +1,61 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/api_extension" +require "elastic_graph/schema_definition/test_support" + +module ElasticGraph + module ProtoIngestion + module SchemaSupport + include ElasticGraph::SchemaDefinition::TestSupport + + def define_proto_schema(**options, &block) + define_proto_schema_results(**options, &block).proto_schema + end + + def define_proto_schema_results(**options, &block) + define_schema( + schema_element_name_form: :snake_case, + extension_modules: [SchemaDefinition::APIExtension], + **options, + &block + ) + end + + def proto_type_def_from(proto, type) + lines = proto.lines + definition_start = /^(?:enum|message) #{Regexp.escape(type)} \{/ + start_indices = lines.each_index.select { |index| definition_start.match?(lines.fetch(index)) } + + if start_indices.size >= 2 + # :nocov: -- only executed when a mistake has been made; causes a failing test. + raise Errors::SchemaError, + "Expected to find 0 or 1 proto type definition for #{type}, but found #{start_indices.size}." + # :nocov: + end + + definition_start_index = start_indices.first + return nil unless definition_start_index + + brace_depth = 0 + result_lines = lines.drop(definition_start_index).each_with_object([]) do |line, collected_lines| + collected_lines << line + structural_content = line.sub(%r{//.*}, "") + brace_depth += structural_content.count("{") - structural_content.count("}") + break collected_lines if brace_depth.zero? + end + + result_lines.join.strip + end + end + + RSpec.configure do |config| + config.include SchemaSupport, :proto_schema + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb index 7f57ae738..23383ab54 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb @@ -7,25 +7,63 @@ # frozen_string_literal: true require "elastic_graph/proto_ingestion/schema_definition/api_extension" -require "elastic_graph/spec_support/schema_definition_helpers" module ElasticGraph module ProtoIngestion module SchemaDefinition RSpec.describe APIExtension do - include_context "SchemaDefinitionHelpers" - - it "can be used as a schema definition extension module, without customizing the schema definition yet" do - with_extension, without_extension = [[APIExtension], []].map do |extension_modules| - define_schema(schema_element_name_form: "snake_case", extension_modules: extension_modules) do |schema| - schema.object_type "Widget" do |t| - t.field "id", "ID!" - t.index "widgets" + it "emits the configured package name" do + proto = define_proto_schema do |s| + s.proto_schema_artifacts package_name: "sales.v1" + + s.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + end + end + + expect(proto).to include("package sales.v1;") + end + + it "maps every built-in scalar to a proto field type" do + field_types = [] + proto = define_proto_schema do |s| + s.object_type "Widget" do |t| + FactoryExtension::BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME.each_key do |type_name| + t.field type_name.downcase, type_name end - end.graphql_schema_string + field_types = t.graphql_fields_by_name.values.map { |field| field.type.name } + t.index "widgets" + end end - expect(with_extension).to eq(without_extension) + expect(field_types).to match_array(FactoryExtension::BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME.keys) + FactoryExtension::BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME.each.with_index(1) do |(type_name, proto_type), field_number| + field_name = Identifier.field_name(type_name.downcase) + expect(proto).to include("#{proto_type} #{field_name} = #{field_number};") + end + end + + it "requires `package_name` to be a non-empty String" do + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: "" + end + }.to raise_error(Errors::SchemaError, a_string_including("`package_name` must be a non-empty String")) + + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: :symbol_package + end + }.to raise_error(Errors::SchemaError, a_string_including("`package_name` must be a non-empty String")) + end + + it "rejects invalid package names when they are configured" do + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: "my-app.events" + end + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`", "my-app.events")) end end end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/factory_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/factory_extension_spec.rb new file mode 100644 index 000000000..1b10b5fc1 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/factory_extension_spec.rb @@ -0,0 +1,60 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/factory_extension" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe FactoryExtension do + it "supports public type definitions without configuration blocks" do + proto = define_proto_schema do |s| + s.enum_type "Status" + s.state.enum_types_by_name.fetch("Status").value "ACTIVE" + + s.interface_type "Named" + s.state.object_types_by_name.fetch("Named").field "name", "String" + + s.object_type "UnconfiguredRecord" + s.state.object_types_by_name.fetch("UnconfiguredRecord").field "id", "ID" + + s.union_type "Entity" + s.state.object_types_by_name.fetch("Entity").subtype "UnconfiguredRecord" + end + + expect(proto).to eq("") + end + + it "validates scalar type definitions without configuration blocks" do + expect { + define_proto_schema do |s| + s.scalar_type "UnconfiguredScalar" + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Protobuf type not configured for scalar type `UnconfiguredScalar`" + )) + end + + it "rejects colliding proto type names as soon as the second type is defined" do + expect { + define_proto_schema do |s| + s.enum_type "package" do |t| + t.value "ACTIVE" + end + + s.object_type "package_" do |t| + t.field "id", "ID" + t.index "packages" + end + end + }.to raise_error(Errors::SchemaError, a_string_including("both map to the same proto type name `package_`")) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/identifier_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/identifier_spec.rb new file mode 100644 index 000000000..d68594090 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/identifier_spec.rb @@ -0,0 +1,55 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/identifier" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe Identifier do + it "escapes reserved keywords" do + expect(Identifier.message_name("package")).to eq("package_") + expect(Identifier.message_name("custom")).to eq("custom") + end + + it "allows keywords in package name segments" do + expect(Identifier.validate_package_name("proto.package.v1")).to eq("proto.package.v1") + end + + it "rejects package names with segments that are not valid protobuf identifiers" do + expect { + Identifier.validate_package_name("my-app.events") + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`", "my-app.events")) + + expect { + Identifier.validate_package_name("myapp..events") + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`")) + + expect { + Identifier.validate_package_name("1myapp.events") + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`")) + + expect { + Identifier.validate_package_name("") + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`")) + + expect { + Identifier.validate_package_name("myapp.events.") + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`")) + end + + it "escapes message, enum, field, and enum value names" do + expect(Identifier.message_name("service")).to eq("service_") + expect(Identifier.enum_name("message")).to eq("message_") + expect(Identifier.field_name("string")).to eq("string_") + expect(Identifier.enum_value_name("stream")).to eq("stream_") + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb new file mode 100644 index 000000000..1a7ed4d78 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb @@ -0,0 +1,56 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension" +require "stringio" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe SchemaArtifactManagerExtension, :in_temp_dir do + it "dumps the proto schema artifact alongside the base artifacts" do + artifact_base_names = artifacts_for(define_indexed_type_schema) + + expect(artifact_base_names).to include(PROTO_SCHEMA_FILE) + end + + it "omits the proto schema artifact when the schema defines no indexed types" do + results = define_proto_schema_results do |s| + s.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + end + end + + artifact_base_names = artifacts_for(results) + + expect(artifact_base_names).not_to include(PROTO_SCHEMA_FILE) + end + + def define_indexed_type_schema + define_proto_schema_results do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + end + end + end + + def artifacts_for(results) + manager = results.state.api.factory.new_schema_artifact_manager( + schema_definition_results: results, + schema_artifacts_directory: "artifacts", + output: ::StringIO.new + ) + + manager.send(:artifacts_from_schema_def).map { |artifact| ::File.basename(artifact.file_name) } + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb new file mode 100644 index 000000000..b1cd46ac4 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb @@ -0,0 +1,91 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/api_extension" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe Schema do + it "returns an empty string when no indexed types are present" do + proto = define_proto_schema do |s| + s.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + end + end + + expect(proto).to eq("") + end + + it "raises when enum values map to duplicate proto value names" do + expect { + define_proto_schema do |s| + s.enum_type "Status" do |t| + t.values "option", "OPTION" + end + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Enum `Status` values `option` and `OPTION`", + "duplicate proto enum value name `STATUS_OPTION`" + )) + end + + it "raises when an enum value conflicts with the generated zero value" do + expect { + define_proto_schema do |s| + s.enum_type "Status" do |t| + t.values "ACTIVE", "UNSPECIFIED" + end + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Enum `Status` value `UNSPECIFIED`", + "conflicts with the generated zero value `STATUS_UNSPECIFIED`" + )) + end + + it "raises when two fields collapse to the same proto field name after keyword escaping" do + expect { + define_proto_schema do |s| + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "string", "String" + t.field "string_", "String" + t.index "accounts" + end + end + }.to raise_error(Errors::SchemaError, a_string_including("duplicate proto field names")) + end + + it "renders a shared enum only once when multiple indexed types reference it" do + proto = define_proto_schema do |s| + s.enum_type "Status" do |t| + t.value "ACTIVE" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "accounts" + end + + s.object_type "User" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "users" + end + end + + expect(proto.scan(/^enum Status \{/).size).to eq(1) + expect(proto_type_def_from(proto, "Account")).to include("Status status = 2;") + expect(proto_type_def_from(proto, "User")).to include("Status status = 2;") + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb new file mode 100644 index 000000000..4206565d5 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb @@ -0,0 +1,288 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/api_extension" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe Schema do + it "generates a proto schema from indexed types" do + proto = define_proto_schema do |s| + s.enum_type "Status" do |t| + t.documentation "The status of an account.\n\nUsed when ingesting accounts." + t.value "ACTIVE" do |v| + v.documentation "The account is active." + end + t.value "INACTIVE" + end + + s.object_type "Address" do |t| + t.field "street", "String" + t.field "city", "String" + end + + s.object_type "Account" do |t| + t.documentation "An account in the system.\n\n" + t.field "id", "ID" do |f| + f.documentation "The account's unique identifier." + end + t.field "status", "Status" + t.field "address", "Address" + t.field "tags", "[String!]!" + t.field "display_name", "String", graphql_only: true + t.index "accounts" + end + end + + expect(proto).to eq(<<~PROTO) + syntax = "proto3"; + + package elasticgraph; + + // The status of an account. + // + // Used when ingesting accounts. + enum Status { + // The default value when no enum value has been explicitly set. Do not use this value. + // See https://protobuf.dev/programming-guides/proto3/#enum-default. + STATUS_UNSPECIFIED = 0; + // The account is active. + STATUS_ACTIVE = 1; + STATUS_INACTIVE = 2; + } + + // An account in the system. + message Account { + // The account's unique identifier. + string id = 1; + Status status = 2; + Address address = 3; + repeated string tags = 4; + } + + message Address { + string street = 1; + string city = 2; + } + PROTO + end + + it "generates oneof wrappers for indexed interface and union types" do + proto = define_proto_schema do |s| + s.object_type "Car" do |t| + t.implements "Vehicle" + t.field "id", "ID" + t.field "doors", "Int" + end + + s.object_type "Bike" do |t| + t.implements "Vehicle" + t.field "id", "ID" + t.field "gears", "Int" + end + + s.interface_type "Vehicle" do |t| + t.field "id", "ID" + t.index "vehicles" + end + + s.object_type "Person" do |t| + t.field "id", "ID" + t.field "name", "String" + end + + s.object_type "Company" do |t| + t.field "id", "ID" + t.field "stock_ticker", "String" + end + + s.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + t.index "inventors" + end + end + + expect(proto_type_def_from(proto, "Vehicle")).to eq(<<~PROTO.strip) + message Vehicle { + oneof value { + Car car = 1; + Bike bike = 2; + } + } + PROTO + expect(proto_type_def_from(proto, "Inventor")).to eq(<<~PROTO.strip) + message Inventor { + oneof value { + Person person = 1; + Company company = 2; + } + } + PROTO + expect(proto_type_def_from(proto, "Car")).to include("string id = 1;", "int32 doors = 2;") + expect(proto_type_def_from(proto, "Bike")).to include("string id = 1;", "int32 gears = 2;") + expect(proto_type_def_from(proto, "Person")).to include("string id = 1;", "string name = 2;") + expect(proto_type_def_from(proto, "Company")).to include("string id = 1;", "string stock_ticker = 2;") + expect(proto_type_def_from(proto, "Missing")).to be_nil + expect(proto).not_to include("__typename") + end + + it "flattens nested interfaces and snake-cases multiword oneof alternatives" do + proto = define_proto_schema do |s| + s.object_type "DeliveryVehicle" do |t| + t.implements "MotorVehicle" + t.field "id", "ID" + end + + s.interface_type "MotorVehicle" do |t| + t.implements "Vehicle" + t.field "id", "ID" + end + + s.interface_type "Vehicle" do |t| + t.field "id", "ID" + t.index "vehicles" + end + end + + expect(proto_type_def_from(proto, "Vehicle")).to eq(<<~PROTO.strip) + message Vehicle { + oneof value { + DeliveryVehicle delivery_vehicle = 1; + } + } + PROTO + expect(proto_type_def_from(proto, "DeliveryVehicle")).to include("string id = 1;") + expect(proto_type_def_from(proto, "MotorVehicle")).to be_nil + end + + it "rejects lists of lists" do + expect { + define_proto_schema do |s| + s.object_type "Matrix" do |t| + t.field "id", "ID" + t.field "values", "[[Float!]!]!" + t.index "matrices" + end + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Field `Matrix.values` has type `[[Float!]!]!`", + "Protocol Buffers cannot represent lists of lists directly", + "at most one list level" + )) + end + + it "uses custom proto scalar mappings" do + proto = define_proto_schema do |s| + s.scalar_type "CustomTimestamp" do |t| + t.mapping type: "date" + t.protobuf type: "int64" + end + + s.object_type "Event" do |t| + t.field "id", "ID" + t.field "occurred_at", "CustomTimestamp" + t.index "events" + end + end + + expect(proto_type_def_from(proto, "Event")).to include("int64 occurred_at = 2;") + end + + it "uses public field names in schema.proto when `name_in_index` differs" do + proto = define_proto_schema do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "display_name", "String", name_in_index: "displayName" + t.index "widgets" + end + end + + widget = proto_type_def_from(proto, "Widget") + expect(widget).to include("string display_name = 2;") + expect(widget).not_to include("displayName") + end + + it "renders independently each time it is called" do + results = define_proto_schema_results do |s| + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + generator = Schema.new( + state: results.state, + all_types: results.send(:all_types), + package_name: "elasticgraph" + ) + + first_generation = generator.to_proto + + second_generation = generator.to_proto + expect(second_generation).to eq(first_generation) + expect(second_generation).not_to be(first_generation) + end + + it "raises when a custom scalar is defined without a protobuf type" do + expect { + define_proto_schema do |s| + s.scalar_type "UnconfiguredScalar" do |t| + t.mapping type: "keyword" + end + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Protobuf type not configured for scalar type `UnconfiguredScalar`.", + "call `protobuf type:" + )) + end + + it "does not require a protobuf type for GraphQL-only scalars" do + proto = define_proto_schema do |s| + s.scalar_type "GraphQLOnlyScalar" do |t| + t.mapping type: "keyword" + t.graphql_only true + end + end + + expect(proto).to eq("") + end + + it "resolves proto field types for built-in scalars that are renamed via `type_name_overrides`" do + proto = define_proto_schema(type_name_overrides: {"JsonSafeLong" => "BigNumber"}) do |s| + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "count", "BigNumber" + t.index "accounts" + end + end + + expect(proto_type_def_from(proto, "Account")).to include("int64 count = 2;") + end + + it "prefixes enum values and escapes proto keywords in generated identifiers" do + proto = define_proto_schema do |s| + s.enum_type "Command" do |t| + t.values "option", "stream" + end + + s.object_type "Request" do |t| + t.field "id", "ID" + t.field "package", "String" + t.field "command", "Command" + t.index "requests" + end + end + + expect(proto_type_def_from(proto, "Command")).to include("COMMAND_OPTION = 1;", "COMMAND_STREAM = 2;") + expect(proto_type_def_from(proto, "Request")).to include("string package_ = 2; // source name: package") + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb deleted file mode 100644 index 558b42477..000000000 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2024 - 2026 Block, Inc. -# -# Use of this source code is governed by an MIT-style -# license that can be found in the LICENSE file or at -# https://opensource.org/licenses/MIT. -# -# frozen_string_literal: true - -require "elastic_graph/proto_ingestion" - -module ElasticGraph - RSpec.describe ProtoIngestion do - it "defines the PROTO_SCHEMA_FILE constant" do - expect(ProtoIngestion::PROTO_SCHEMA_FILE).to eq("schema.proto") - end - - it "defines the PROTO_FIELD_NUMBERS_FILE constant" do - expect(ProtoIngestion::PROTO_FIELD_NUMBERS_FILE).to eq("proto_field_numbers.yaml") - end - end -end From f2ef14ebd28e88639f14877ad5fdc87ea693cddc Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 8 Jul 2026 18:19:37 -0500 Subject: [PATCH 2/7] Pin protobuf field and enum value numbers in a sidecar artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field and enum value numbers were assigned sequentially in definition order, so reordering or removing a field renumbered everything after it — breaking wire compatibility with previously serialized data. `schema_artifacts:dump` now reads and writes a `proto_field_numbers.yaml` sidecar artifact: - Existing numbers stay fixed even if field order changes; new fields get the next available numbers. - `schema.proto` keeps the public GraphQL field names while the sidecar stores private `name_in_index` overrides. - A field renamed with `field.renamed_from` reuses its existing number under the new public name. - Enum value numbers are pinned in an `enums` section; removed values keep their numbers reserved so they are never reused (`0` remains the generated `*_UNSPECIFIED` value). - Mappings are validated: collisions raise clear errors, and non-integer numbers are rejected instead of letting `Integer()` silently truncate. `configure_proto_field_number_mappings` allows tests and advanced callers to seed mappings without an artifact file. --- elasticgraph-proto_ingestion/README.md | 48 ++- .../lib/elastic_graph/proto_ingestion.rb | 3 + .../schema_definition/api_extension.rb | 12 + .../proto_ingestion_state.rb | 2 +- .../schema_definition/results_extension.rb | 30 +- .../schema_definition/schema.rb | 349 ++++++++++++++- .../schema_artifact_manager_extension.rb | 25 ++ .../schema_elements/enum_type_extension.rb | 12 +- .../object_interface_and_union_extension.rb | 26 +- .../schema_definition/state_extension.rb | 2 +- .../sig/elastic_graph/proto_ingestion.rbs | 1 + .../schema_definition/api_extension.rbs | 1 + .../proto_ingestion_state.rbs | 3 +- .../schema_definition/results_extension.rbs | 2 + .../schema_definition/schema.rbs | 44 +- .../schema_artifact_manager_extension.rbs | 2 + .../schema_elements/enum_type_extension.rbs | 4 +- .../object_interface_and_union_extension.rbs | 6 +- .../schema_definition/rake_tasks_spec.rb | 43 ++ .../schema_artifact_manager_extension_spec.rb | 49 ++- .../schema_edge_cases_spec.rb | 405 ++++++++++++++++++ .../schema_definition/schema_spec.rb | 332 +++++++++++++- 22 files changed, 1359 insertions(+), 42 deletions(-) diff --git a/elasticgraph-proto_ingestion/README.md b/elasticgraph-proto_ingestion/README.md index ba6ffe1b6..fafefb9ff 100644 --- a/elasticgraph-proto_ingestion/README.md +++ b/elasticgraph-proto_ingestion/README.md @@ -72,7 +72,10 @@ ElasticGraph.define_schema do |schema| end ``` -After running `bundle exec rake schema_artifacts:dump`, ElasticGraph will generate `schema.proto`. +After running `bundle exec rake schema_artifacts:dump`, ElasticGraph will generate: + +- `schema.proto` +- `proto_field_numbers.yaml` ## Schema Definition API @@ -93,6 +96,49 @@ ElasticGraph.define_schema do |schema| end ``` +### Stable Field Numbers + +`schema_artifacts:dump` automatically reads and writes `proto_field_numbers.yaml` +in the schema artifacts directory. Existing numbers stay fixed even if field order +changes, and new fields get the next available numbers. Field numbers follow protobuf's +rules: they must be between 1 and 536,870,911, and the protobuf-reserved 19000-19999 +range is never allocated and is rejected in mappings. + +Alternatives inside generated interface and union `oneof` blocks use the same stable +message-field mappings, so adding or removing a concrete subtype does not renumber the +remaining alternatives. + +`schema.proto` always uses the public GraphQL field names. When a field uses a +different `name_in_index`, the sidecar YAML stores that override privately: + +```yaml +messages: + Widget: + fields: + id: 1 + display_name: + field_number: 2 + name_in_index: displayName +``` + +If a field is renamed with `field.renamed_from`, `elasticgraph-proto_ingestion` reuses the +existing field number under the new public field name. + +### Stable Enum Value Numbers + +Enum value numbers are pinned the same way, in an `enums` section of the sidecar. Existing +values keep their numbers when other values are added or removed, new values get the next +available numbers, and removed values keep their numbers reserved so they are never reused +(number `0` is always the generated `*_UNSPECIFIED` value): + +```yaml +enums: + WidgetColor: + values: + RED: 1 + BLUE: 2 +``` + ## Type Mappings The generated `schema.proto` uses these built-in scalar mappings: diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb index e48ad46e2..df5f14429 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb @@ -11,5 +11,8 @@ module ElasticGraph module ProtoIngestion # The name of the generated Protocol Buffers schema file. PROTO_SCHEMA_FILE = "schema.proto" + + # The name of the generated proto field-number mapping file. + PROTO_FIELD_NUMBERS_FILE = "proto_field_numbers.yaml" end end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb index e0a987b47..fa692a5ec 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb @@ -50,6 +50,18 @@ def proto_schema_artifacts(package_name:) nil end + # Configures proto field-number mappings directly from a hash. + # Useful for tests and advanced use cases where mappings are sourced outside artifacts. + # When artifacts are dumped, mappings from the existing `proto_field_numbers.yaml` artifact + # are loaded automatically; this method does not need to be called in that case. + # + # @param proto_field_number_mappings [Hash] + # @return [void] + def configure_proto_field_number_mappings(proto_field_number_mappings) + proto_ingestion_state.field_number_mappings = proto_field_number_mappings + nil + end + private # Returns this gem's state container. Centralizes the Steep cast that's needed because diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb index 5dd914cda..e5c4d921a 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb @@ -14,7 +14,7 @@ module SchemaDefinition # Holds the proto ingestion extension's schema definition state. # # @private - class ProtoIngestionState < ::Struct.new(:package_name) + class ProtoIngestionState < ::Struct.new(:package_name, :field_number_mappings) def initialize(...) super @type_name_by_proto_name = {} diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb index 50fa81213..6e9f3be3d 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb @@ -20,19 +20,31 @@ def proto_schema @proto_schema ||= protobuf_schema_generator.to_proto end + # Returns proto field-number mappings suitable for artifact storage. + # + # @return [Hash] + def proto_field_number_mappings + # Ensure generation has occurred before reading mappings from the generator. + proto_schema + protobuf_schema_generator.field_number_mappings_for_artifact + end + private def protobuf_schema_generator - # The cast is needed because Steep can't see the `extend(StateExtension)` applied at - # runtime in {APIExtension.extended}. - extension_state = state # : ElasticGraph::SchemaDefinition::State & StateExtension - ingestion_state = extension_state.proto_ingestion_state + @protobuf_schema_generator ||= begin + # The cast is needed because Steep can't see the `extend(StateExtension)` applied at + # runtime in {APIExtension.extended}. + extension_state = state # : ElasticGraph::SchemaDefinition::State & StateExtension + ingestion_state = extension_state.proto_ingestion_state - Schema.new( - state: extension_state, - all_types: all_types, - package_name: ingestion_state.package_name - ) + Schema.new( + state: extension_state, + all_types: all_types, + package_name: ingestion_state.package_name, + proto_field_number_mappings: ingestion_state.field_number_mappings + ) + end end end end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb index 11647e1d2..5b43274f4 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb @@ -6,6 +6,7 @@ # # frozen_string_literal: true +require "elastic_graph/errors" require "elastic_graph/proto_ingestion/schema_definition/identifier" require "elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension" require "elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension" @@ -16,13 +17,38 @@ module ProtoIngestion module SchemaDefinition # Builds a `proto3` schema string from an ElasticGraph schema definition. class Schema + # Internal representation of a stored field-number mapping. + # + # @!attribute [r] field_number + # @return [Integer] + # @!attribute [r] name_in_index + # @return [String] + FieldNumberMapping = ::Data.define(:field_number, :name_in_index) + + # The largest field number protobuf allows (2^29 - 1). + MAX_FIELD_NUMBER = 536_870_911 + # Field numbers protobuf reserves for its own implementation; they may not be used as field tags. + RESERVED_FIELD_NUMBER_RANGE = 19_000..19_999 + # The largest enum value number protobuf allows (the int32 maximum). + MAX_ENUM_VALUE_NUMBER = 2_147_483_647 + # @param state [ElasticGraph::SchemaDefinition::State] # @param all_types [Array] # @param package_name [String] - def initialize(state:, all_types:, package_name:) + # @param proto_field_number_mappings [Hash] + def initialize( + state:, + all_types:, + package_name:, + proto_field_number_mappings: {} + ) @state = state @all_types = all_types @package_name = Identifier.validate_package_name(package_name) + @proto_field_number_mappings_by_message = normalize_proto_field_number_mappings(proto_field_number_mappings) + @proto_enum_value_numbers_by_enum = normalize_proto_enum_value_number_mappings(proto_field_number_mappings) + @used_field_numbers_by_message = {} + @used_enum_value_numbers_by_enum = {} end # Renders the schema as a valid `proto3` file. @@ -41,6 +67,88 @@ def to_proto sections.join("\n\n") + "\n" end + # Exposes normalized field-number and enum-value-number mappings for writing to artifact YAML. + # + # @return [Hash] + def field_number_mappings_for_artifact + { + "messages" => @proto_field_number_mappings_by_message + .sort_by { |message_name, _| message_name } + .to_h do |message_name, field_numbers| + [message_name, { + "fields" => field_numbers.sort_by { |field_name, mapping| [mapping.field_number, field_name] }.to_h do |field_name, mapping| + artifact_mapping = + if mapping.name_in_index == field_name + mapping.field_number + else + { + "field_number" => mapping.field_number, + "name_in_index" => mapping.name_in_index + } + end + + [field_name, artifact_mapping] + end + }] + end, + "enums" => @proto_enum_value_numbers_by_enum + .sort_by { |enum_name, _| enum_name } + .to_h do |enum_name, value_numbers| + [enum_name, { + "values" => value_numbers.sort_by { |value_name, number| [number, value_name] }.to_h + }] + end + } + end + + # Returns the stable protobuf number for a message field. + # + # @api private + def field_number_for(message_name:, type_name:, public_field_name:, name_in_index:) + mappings_for_message = @proto_field_number_mappings_by_message[message_name] ||= {} + used_numbers = @used_field_numbers_by_message[message_name] ||= ::Set.new(mappings_for_message.values.map(&:field_number)) + + mapping = mappings_for_message.fetch(public_field_name) do + migrate_renamed_field_mapping(mappings_for_message, type_name: type_name, public_field_name: public_field_name) || begin + next_field_number = next_available_field_number(used_numbers) + used_numbers << next_field_number + FieldNumberMapping.new(field_number: next_field_number, name_in_index: name_in_index) + end + end + + mapping = FieldNumberMapping.new(field_number: mapping.field_number, name_in_index: name_in_index) if mapping.name_in_index != name_in_index + mappings_for_message[public_field_name] = mapping + + duplicate_field_name = mappings_for_message.find do |mapped_field_name, mapped_field_number| + mapped_field_name != public_field_name && mapped_field_number.field_number == mapping.field_number + end&.first + + if duplicate_field_name + raise Errors::SchemaError, "Protobuf field-number mapping collision in message `#{message_name}`: " \ + "`#{duplicate_field_name}` and `#{public_field_name}` are both mapped to field number #{mapping.field_number}." + end + + mapping.field_number + end + + # Returns the stable protobuf numbers for an enum's values. + # + # @api private + def enum_value_numbers_for(enum_name, value_names) + value_numbers = @proto_enum_value_numbers_by_enum[enum_name] ||= {} + used_numbers = used_enum_value_numbers_for(enum_name, value_numbers) + + value_names.to_h do |value_name| + number = value_numbers.fetch(value_name) do + next_available_enum_value_number(used_numbers).tap do |allocated| + used_numbers << allocated + value_numbers[value_name] = allocated + end + end + [value_name, number] + end + end + private # Selects the indexed root types and every type transitively referenced by their protobuf @@ -63,9 +171,246 @@ def proto_types def render_definitions(types) types .sort_by { |type| [(type.proto_definition_kind == :enum) ? 0 : 1, type.proto_name] } - .filter_map(&:to_proto) + .filter_map do |type| + type.proto_definition_kind ? type.to_proto(self) : type.to_proto + end .join("\n\n") end + + def valid_field_number?(number) + number.between?(1, MAX_FIELD_NUMBER) && !RESERVED_FIELD_NUMBER_RANGE.cover?(number) + end + + # Returns the smallest valid field tag not yet present in `used_numbers`, skipping the + # protobuf-reserved 19000..19999 range. Callers maintain the set (numbers loaded from a + # previously dumped artifact plus numbers allocated so far) so that allocation is + # constant-time per candidate instead of rescanning all mappings. + def next_available_field_number(used_numbers) + candidate = 1 + candidate += 1 while used_numbers.include?(candidate) || RESERVED_FIELD_NUMBER_RANGE.cover?(candidate) + candidate + end + + # Returns the smallest positive enum value number not yet present in `used_numbers`. + # Unlike field tags, enum value numbers have no protobuf-reserved range. + def next_available_enum_value_number(used_numbers) + candidate = 1 + candidate += 1 while used_numbers.include?(candidate) + candidate + end + + def used_enum_value_numbers_for(enum_name, value_numbers) + @used_enum_value_numbers_by_enum[enum_name] ||= begin + duplicated_numbers = value_numbers.group_by { |_, number| number }.select { |_, entries| entries.size > 1 } + + duplicated_numbers.each_value do |entries| + value_names = entries.map(&:first).sort.map { |name| "`#{name}`" }.join(" and ") + raise Errors::SchemaError, "Protobuf enum value-number mapping collision in enum `#{enum_name}`: " \ + "#{value_names} are both mapped to number #{entries.first.last}." + end + + ::Set.new(value_numbers.values) + end + end + + def migrate_renamed_field_mapping(mappings_for_message, type_name:, public_field_name:) + renames_for_type = renamed_public_field_names_by_type_name.fetch(type_name) { return nil } + old_field_names = renames_for_type.fetch(public_field_name) { return nil } + + old_field_names.each do |old_field_name| + return mappings_for_message.delete(old_field_name) if mappings_for_message.key?(old_field_name) + end + + nil + end + + def normalize_proto_field_number_mappings(raw_mappings) + return {} if raw_mappings.nil? + unless raw_mappings.is_a?(Hash) + raise Errors::SchemaError, "Protobuf field-number mappings must be a Hash, got: #{raw_mappings.class}." + end + + messages_hash = + if raw_mappings.key?("messages") + raw_mappings.fetch("messages") + elsif raw_mappings.key?(:messages) + raw_mappings.fetch(:messages) + elsif raw_mappings.key?("enums") || raw_mappings.key?(:enums) + # The hash uses the sectioned artifact format but only maps enum value numbers. + {} # : ::Hash[untyped, untyped] + else + raw_mappings + end + + unless messages_hash.is_a?(Hash) + raise Errors::SchemaError, "Protobuf field-number mappings must have a `messages` Hash." + end + + normalized = {} # : ::Hash[::String, fieldNumberMappingsByFieldName] + + messages_hash.each do |message_name, field_numbers| + unless field_numbers.is_a?(Hash) + raise Errors::SchemaError, "Field-number mapping for message `#{message_name}` must be a Hash." + end + + normalized_fields = + if field_numbers.key?("fields") + field_numbers.fetch("fields") + elsif field_numbers.key?(:fields) + field_numbers.fetch(:fields) + else + field_numbers + end + + unless normalized_fields.is_a?(Hash) + raise Errors::SchemaError, "Field-number mapping for message `#{message_name}` must contain a `fields` Hash." + end + + normalized_message_name = message_name.to_s + normalized_field_numbers = {} # : fieldNumberMappingsByFieldName + + normalized_fields.each do |field_name, field_number_or_mapping| + normalized_field_name = field_name.to_s + normalized_field_number, normalized_name_in_index = normalize_field_number_mapping_entry( + normalized_message_name, + normalized_field_name, + field_number_or_mapping + ) + + unless valid_field_number?(normalized_field_number) + raise Errors::SchemaError, "Field-number mapping for `#{normalized_message_name}.#{normalized_field_name}` " \ + "must be a valid protobuf field number (1 to #{MAX_FIELD_NUMBER}, excluding the reserved " \ + "#{RESERVED_FIELD_NUMBER_RANGE.begin}-#{RESERVED_FIELD_NUMBER_RANGE.end} range), got: #{field_number_or_mapping.inspect}." + end + + normalized_field_numbers[normalized_field_name] = FieldNumberMapping.new( + field_number: normalized_field_number, + name_in_index: normalized_name_in_index + ) + end + + normalized[normalized_message_name] = normalized_field_numbers + end + + normalized + end + + def normalize_proto_enum_value_number_mappings(raw_mappings) + return {} if raw_mappings.nil? + + enums_hash = + if raw_mappings.is_a?(Hash) && raw_mappings.key?("enums") + raw_mappings.fetch("enums") + elsif raw_mappings.is_a?(Hash) && raw_mappings.key?(:enums) + raw_mappings.fetch(:enums) + else + return {} + end + + unless enums_hash.is_a?(Hash) + raise Errors::SchemaError, "Protobuf enum value-number mappings must have an `enums` Hash." + end + + normalized = {} # : ::Hash[::String, ::Hash[::String, ::Integer]] + + enums_hash.each do |enum_name, value_numbers| + values_hash = + if value_numbers.is_a?(Hash) && value_numbers.key?("values") + value_numbers.fetch("values") + elsif value_numbers.is_a?(Hash) && value_numbers.key?(:values) + value_numbers.fetch(:values) + else + value_numbers + end + + unless values_hash.is_a?(Hash) + raise Errors::SchemaError, "Enum value-number mapping for enum `#{enum_name}` must contain a `values` Hash." + end + + normalized_enum_name = enum_name.to_s + normalized_value_numbers = {} # : ::Hash[::String, ::Integer] + + values_hash.each do |value_name, value_number| + normalized_number = integral_number(value_number) + + if normalized_number.nil? || normalized_number <= 0 || normalized_number > MAX_ENUM_VALUE_NUMBER + raise Errors::SchemaError, "Enum value-number mapping for `#{normalized_enum_name}.#{value_name}` " \ + "must be a positive integer no greater than #{MAX_ENUM_VALUE_NUMBER} " \ + "(0 is reserved for the `_UNSPECIFIED` value), got: #{value_number.inspect}." + end + + normalized_value_numbers[value_name.to_s] = normalized_number + end + + normalized[normalized_enum_name] = normalized_value_numbers + end + + normalized + end + + def normalize_field_number_mapping_entry(message_name, field_name, field_number_or_mapping) + if field_number_or_mapping.is_a?(Hash) + raw_field_number = + if field_number_or_mapping.key?("field_number") + field_number_or_mapping.fetch("field_number") + elsif field_number_or_mapping.key?(:field_number) + field_number_or_mapping.fetch(:field_number) + else + raise Errors::SchemaError, "Field-number mapping for `#{message_name}.#{field_name}` must include `field_number`." + end + + raw_name_in_index = + if field_number_or_mapping.key?("name_in_index") + field_number_or_mapping.fetch("name_in_index") + elsif field_number_or_mapping.key?(:name_in_index) + field_number_or_mapping.fetch(:name_in_index) + else + field_name + end + + unless raw_name_in_index.is_a?(String) || raw_name_in_index.is_a?(Symbol) + raise Errors::SchemaError, "Field-number mapping for `#{message_name}.#{field_name}` " \ + "must use a String or Symbol `name_in_index`, got: #{raw_name_in_index.inspect}." + end + + [integral_field_number(message_name, field_name, raw_field_number), raw_name_in_index.to_s] + else + [integral_field_number(message_name, field_name, field_number_or_mapping), field_name] + end + end + + def integral_field_number(message_name, field_name, raw_field_number) + number = integral_number(raw_field_number) + + if number.nil? + raise Errors::SchemaError, "Field-number mapping for `#{message_name}.#{field_name}` " \ + "must be an integer, got: #{raw_field_number.inspect}." + end + + number + end + + # Coerces a raw mapping number to an Integer without truncating: only `Integer` values and + # strings that parse as integers are accepted; anything else (including floats, which + # `Kernel#Integer` would silently truncate) returns `nil`. + def integral_number(raw) + case raw + when ::Integer then raw + when ::String then raw.match?(/\A-?\d+\z/) ? Integer(raw) : nil + end + end + + def renamed_public_field_names_by_type_name + @renamed_public_field_names_by_type_name ||= @state.renamed_fields_by_type_name_and_old_field_name.to_h do |type_name, old_to_new| + current_to_old = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::String]] + + old_to_new.each do |old_field_name, renamed_field| + current_to_old[renamed_field.name] << old_field_name + end + + [type_name, current_to_old] + end + end end end end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rb index 352c2df84..2a0bb322d 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/proto_ingestion" +require "yaml" module ElasticGraph module ProtoIngestion @@ -20,11 +21,21 @@ module SchemaArtifactManagerExtension # Overrides the base `artifacts_from_schema_def` method to add proto artifacts. def artifacts_from_schema_def + protobuf_load_existing_field_number_mappings + base_artifacts = super proto_schema = protobuf_schema_definition_results.proto_schema return base_artifacts if proto_schema.empty? base_artifacts + [ + new_yaml_artifact( + PROTO_FIELD_NUMBERS_FILE, + protobuf_schema_definition_results.proto_field_number_mappings, + extra_comment_lines: [ + "This file reserves protobuf field and enum value numbers to keep them stable over time.", + "Do not renumber existing entries." + ] + ), new_raw_artifact(PROTO_SCHEMA_FILE, proto_schema.chomp, comment_prefix: "//") ] end @@ -35,6 +46,20 @@ def artifacts_from_schema_def def protobuf_schema_definition_results schema_definition_results # : ElasticGraph::SchemaDefinition::Results & ResultsExtension end + + def proto_ingestion_state + extension_state = protobuf_schema_definition_results.state # : ElasticGraph::SchemaDefinition::State & StateExtension + extension_state.proto_ingestion_state + end + + # Seeds the schema generator with the field-number mappings from the previously dumped + # artifact (if any) so that field numbers remain stable across dumps. + def protobuf_load_existing_field_number_mappings + full_path = ::File.join(@schema_artifacts_directory, PROTO_FIELD_NUMBERS_FILE) + return unless ::File.exist?(full_path) + + proto_ingestion_state.field_number_mappings = ::YAML.safe_load_file(full_path, aliases: false) + end end end end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb index 37c2fbfe4..2652b66e5 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb @@ -48,8 +48,8 @@ def value(value_name, &block) # Renders this enum's protobuf definition. # # @return [String] - def to_proto - render_proto_enum + def to_proto(schema) + render_proto_enum(schema) end # Returns the kind used to order this definition in a protobuf schema. @@ -82,8 +82,10 @@ def configure_derived_scalar_type(scalar_type) private - def render_proto_enum + def render_proto_enum(schema) values = values_by_name.values + value_numbers = schema.enum_value_numbers_for(proto_name, values_by_name.keys) + zero_value_name = "#{enum_value_prefix}_UNSPECIFIED" lines = ProtoDocumentation.comment_lines_for(doc_comment) @@ -91,10 +93,10 @@ def render_proto_enum lines << " // The default value when no enum value has been explicitly set. Do not use this value." lines << " // See https://protobuf.dev/programming-guides/proto3/#enum-default." lines << " #{zero_value_name} = 0;" - values.each.with_index(1) do |raw_value, number| + values.each do |raw_value| value = raw_value # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension lines.concat(ProtoDocumentation.comment_lines_for(value.doc_comment, indent: " ")) - lines << " #{value.proto_name(enum_value_prefix)} = #{number};" + lines << " #{value.proto_name(enum_value_prefix)} = #{value_numbers.fetch(value.name)};" end lines << "}" lines.join("\n") diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb index 9e6ded865..1f3ab6e44 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb @@ -20,8 +20,8 @@ module ObjectInterfaceAndUnionExtension # Renders this type's protobuf message definition. # # @return [String] - def to_proto - render_proto_message(proto_name) + def to_proto(schema) + render_proto_message(schema, proto_name) end # Returns the kind used to order this definition in a protobuf schema. @@ -54,8 +54,8 @@ def proto_name private - def render_proto_message(message_name) - return render_proto_oneof(message_name) if abstract? + def render_proto_message(schema, message_name) + return render_proto_oneof(schema, message_name) if abstract? fields = proto_fields field_names = fields.map { |_, field| Identifier.field_name(field.name) } @@ -66,9 +66,15 @@ def render_proto_message(message_name) lines = ProtoDocumentation.comment_lines_for(doc_comment) lines << "message #{message_name} {" - fields.each.with_index(1) do |(schema_field, field), field_number| + fields.each do |schema_field, field| field_name = Identifier.field_name(field.name) repeated, field_type = proto_field_type_for(field.type, context_field_name: field.name) + field_number = schema.field_number_for( + message_name: message_name, + type_name: name, + public_field_name: schema_field.name, + name_in_index: field.name_in_index + ) label = "repeated " if repeated line = " #{label}#{field_type} #{field_name} = #{field_number};" line += " // source name: #{field.name}" if field_name != field.name @@ -79,15 +85,21 @@ def render_proto_message(message_name) lines.join("\n") end - def render_proto_oneof(message_name) + def render_proto_oneof(schema, message_name) # @type var abstract_type: ::ElasticGraph::SchemaDefinition::Mixins::HasSubtypes abstract_type = _ = self lines = ProtoDocumentation.comment_lines_for(doc_comment) lines << "message #{message_name} {" lines << " oneof value {" - abstract_type.recursively_resolve_subtypes.each.with_index(1) do |subtype, field_number| + abstract_type.recursively_resolve_subtypes.each do |subtype| subtype_name = (_ = subtype).proto_name field_name = Identifier.field_name(Support::Casing.to_upper_snake(subtype_name).downcase) + field_number = schema.field_number_for( + message_name: message_name, + type_name: name, + public_field_name: field_name, + name_in_index: field_name + ) lines << " #{subtype_name} #{field_name} = #{field_number};" end lines << " }" diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb index f59e96343..891ae5a2d 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb @@ -21,7 +21,7 @@ module StateExtension def self.extended(state) state.instance_variable_set( :@proto_ingestion_state, - ProtoIngestionState.new(package_name: "elasticgraph") + ProtoIngestionState.new(package_name: "elasticgraph", field_number_mappings: {}) ) end end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs index 7cb944875..81cc74611 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs @@ -1,5 +1,6 @@ module ElasticGraph module ProtoIngestion PROTO_SCHEMA_FILE: ::String + PROTO_FIELD_NUMBERS_FILE: ::String end end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs index e7bb15e78..e8ac37ee5 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs @@ -4,6 +4,7 @@ module ElasticGraph module APIExtension: ::ElasticGraph::SchemaDefinition::API def self.extended: (::ElasticGraph::SchemaDefinition::API & APIExtension) -> void def proto_schema_artifacts: (package_name: ::String) -> void + def configure_proto_field_number_mappings: (untyped) -> void private diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs index 964206a09..5947f7c14 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs @@ -3,8 +3,9 @@ module ElasticGraph module SchemaDefinition class ProtoIngestionStateSupertype attr_accessor package_name: ::String + attr_accessor field_number_mappings: untyped - def initialize: (package_name: ::String) -> void + def initialize: (package_name: ::String, field_number_mappings: untyped) -> void end class ProtoIngestionState < ProtoIngestionStateSupertype diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/results_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/results_extension.rbs index 96de3ae03..2d5339a10 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/results_extension.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/results_extension.rbs @@ -3,12 +3,14 @@ module ElasticGraph module SchemaDefinition module ResultsExtension : ::ElasticGraph::SchemaDefinition::Results def proto_schema: () -> ::String + def proto_field_number_mappings: () -> untyped private def protobuf_schema_generator: () -> Schema @proto_schema: ::String? + @protobuf_schema_generator: Schema? end end end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs index 7ff135db9..7e7f15f70 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs @@ -2,22 +2,64 @@ module ElasticGraph module ProtoIngestion module SchemaDefinition class Schema + class FieldNumberMapping + attr_reader field_number: ::Integer + attr_reader name_in_index: ::String + + def self.new: (field_number: ::Integer, name_in_index: ::String) -> instance + end + + type fieldNumberMappingsByFieldName = ::Hash[::String, FieldNumberMapping] + + MAX_FIELD_NUMBER: ::Integer + RESERVED_FIELD_NUMBER_RANGE: ::Range[::Integer] + MAX_ENUM_VALUE_NUMBER: ::Integer + @state: ::ElasticGraph::SchemaDefinition::State @all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] @package_name: ::String + @proto_field_number_mappings_by_message: ::Hash[::String, fieldNumberMappingsByFieldName] + @proto_enum_value_numbers_by_enum: ::Hash[::String, ::Hash[::String, ::Integer]] + @used_field_numbers_by_message: ::Hash[::String, ::Set[::Integer]] + @used_enum_value_numbers_by_enum: ::Hash[::String, ::Set[::Integer]] + @renamed_public_field_names_by_type_name: ::Hash[::String, ::Hash[::String, ::Array[::String]]]? def initialize: ( state: ::ElasticGraph::SchemaDefinition::State, all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType], - package_name: ::String + package_name: ::String, + ?proto_field_number_mappings: untyped ) -> void def to_proto: () -> ::String + def field_number_mappings_for_artifact: () -> ::Hash[::String, untyped] + def field_number_for: ( + message_name: ::String, + type_name: ::String, + public_field_name: ::String, + name_in_index: ::String + ) -> ::Integer + def enum_value_numbers_for: (::String, ::Array[::String]) -> ::Hash[::String, ::Integer] private def proto_types: () -> ::Array[untyped] def render_definitions: (::Array[untyped] types) -> ::String + def valid_field_number?: (::Integer) -> bool + def next_available_field_number: (::Set[::Integer]) -> ::Integer + def next_available_enum_value_number: (::Set[::Integer]) -> ::Integer + def used_enum_value_numbers_for: (::String, ::Hash[::String, ::Integer]) -> ::Set[::Integer] + def migrate_renamed_field_mapping: ( + fieldNumberMappingsByFieldName, + type_name: ::String, + public_field_name: ::String + ) -> FieldNumberMapping? + def normalize_proto_field_number_mappings: (untyped) -> ::Hash[::String, fieldNumberMappingsByFieldName] + def normalize_proto_enum_value_number_mappings: (untyped) -> ::Hash[::String, ::Hash[::String, ::Integer]] + def normalize_field_number_mapping_entry: (::String, ::String, untyped) -> [::Integer, ::String] + def integral_field_number: (::String, ::String, untyped) -> ::Integer + def integral_number: (untyped) -> ::Integer? + def renamed_public_field_names_by_type_name: () -> ::Hash[::String, ::Hash[::String, ::Array[::String]]] end end end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rbs index 4298c1a99..91342e799 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rbs @@ -6,6 +6,8 @@ module ElasticGraph def artifacts_from_schema_def: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]] def protobuf_schema_definition_results: () -> (::ElasticGraph::SchemaDefinition::Results & ResultsExtension) + def proto_ingestion_state: () -> ProtoIngestionState + def protobuf_load_existing_field_number_mappings: () -> void end end end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs index 5d7c62c61..5e47f4c57 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs @@ -7,14 +7,14 @@ module ElasticGraph def value: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension) -> void } -> void def proto_name: () -> ::String - def to_proto: () -> ::String + def to_proto: (Schema) -> ::String def proto_definition_kind: () -> :enum def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] def configure_derived_scalar_type: (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType) -> void private - def render_proto_enum: () -> ::String + def render_proto_enum: (Schema) -> ::String def enum_value_prefix: () -> ::String end end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs index 3b2bf3f3f..b1d562c7f 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs @@ -6,14 +6,14 @@ module ElasticGraph @proto_name: ::String? def proto_name: () -> ::String - def to_proto: () -> ::String + def to_proto: (Schema) -> ::String def proto_definition_kind: () -> :message def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] private - def render_proto_message: (::String message_name) -> ::String - def render_proto_oneof: (::String message_name) -> ::String + def render_proto_message: (Schema, ::String message_name) -> ::String + def render_proto_oneof: (Schema, ::String message_name) -> ::String def proto_fields: () -> ::Array[[ ::ElasticGraph::SchemaDefinition::SchemaElements::Field, ::ElasticGraph::SchemaDefinition::Indexing::Field diff --git a/elasticgraph-proto_ingestion/spec/integration/elastic_graph/proto_ingestion/schema_definition/rake_tasks_spec.rb b/elasticgraph-proto_ingestion/spec/integration/elastic_graph/proto_ingestion/schema_definition/rake_tasks_spec.rb index 66fae144e..4f702ed4b 100644 --- a/elasticgraph-proto_ingestion/spec/integration/elastic_graph/proto_ingestion/schema_definition/rake_tasks_spec.rb +++ b/elasticgraph-proto_ingestion/spec/integration/elastic_graph/proto_ingestion/schema_definition/rake_tasks_spec.rb @@ -9,6 +9,7 @@ require "elastic_graph/proto_ingestion" require "elastic_graph/proto_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" +require "yaml" module ElasticGraph module ProtoIngestion @@ -52,6 +53,44 @@ module SchemaDefinition expect(output.lines).to include(a_string_including("already up to date", PROTO_SCHEMA_FILE)) }.to maintain { read_artifact(PROTO_SCHEMA_FILE) } end + + it "can persist and reuse proto field-number mappings from an artifact file" do + write_proto_schema(table_defs: <<~EOS) + s.object_type "Product" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "products" + end + EOS + + run_rake_with_proto("schema_artifacts:dump") + + expect(read_artifact(PROTO_FIELD_NUMBERS_FILE)).not_to be_nil + expect(parsed_proto_field_numbers).to eq({ + "enums" => {}, + "messages" => { + "Product" => { + "fields" => { + "id" => 1, + "name" => 2 + } + } + } + }) + + write_proto_schema(table_defs: <<~EOS) + s.object_type "Product" do |t| + t.field "name", "String" + t.field "id", "ID" + t.index "products" + end + EOS + + run_rake_with_proto("schema_artifacts:dump") + + expect(read_artifact(PROTO_SCHEMA_FILE)).to include("string name = 2;") + expect(read_artifact(PROTO_SCHEMA_FILE)).to include("string id = 1;") + end end private @@ -81,6 +120,10 @@ def read_artifact(name) path = File.join("config", "schema", "artifacts", name) File.read(path) if File.exist?(path) end + + def parsed_proto_field_numbers + ::YAML.safe_load(read_artifact(PROTO_FIELD_NUMBERS_FILE)) + end end end end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb index 1a7ed4d78..0302e8b65 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb @@ -7,19 +7,20 @@ # frozen_string_literal: true require "elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension" +require "fileutils" require "stringio" module ElasticGraph module ProtoIngestion module SchemaDefinition RSpec.describe SchemaArtifactManagerExtension, :in_temp_dir do - it "dumps the proto schema artifact alongside the base artifacts" do + it "dumps proto artifacts alongside the base artifacts" do artifact_base_names = artifacts_for(define_indexed_type_schema) - expect(artifact_base_names).to include(PROTO_SCHEMA_FILE) + expect(artifact_base_names).to include(PROTO_SCHEMA_FILE, PROTO_FIELD_NUMBERS_FILE) end - it "omits the proto schema artifact when the schema defines no indexed types" do + it "omits proto artifacts when the schema defines no indexed types" do results = define_proto_schema_results do |s| s.object_type "Point" do |t| t.field "x", "Float" @@ -29,7 +30,47 @@ module SchemaDefinition artifact_base_names = artifacts_for(results) - expect(artifact_base_names).not_to include(PROTO_SCHEMA_FILE) + expect(artifact_base_names).not_to include(PROTO_SCHEMA_FILE, PROTO_FIELD_NUMBERS_FILE) + end + + it "seeds proto generation with the field-number mappings from a previously dumped artifact" do + ::FileUtils.mkdir_p("artifacts") + ::File.write(::File.join("artifacts", PROTO_FIELD_NUMBERS_FILE), <<~YAML) + messages: + Widget: + fields: + id: 7 + YAML + + results = define_indexed_type_schema + artifacts_for(results) + + expect(results.proto_schema).to include("string id = 7;") + end + + it "seeds proto generation with the enum value numbers from a previously dumped artifact" do + ::FileUtils.mkdir_p("artifacts") + ::File.write(::File.join("artifacts", PROTO_FIELD_NUMBERS_FILE), <<~YAML) + enums: + Status: + values: + INACTIVE: 5 + YAML + + results = define_proto_schema_results do |s| + s.enum_type "Status" do |t| + t.values "ACTIVE", "INACTIVE" + end + + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "widgets" + end + end + artifacts_for(results) + + expect(results.proto_schema).to include("STATUS_ACTIVE = 1;", "STATUS_INACTIVE = 5;") end def define_indexed_type_schema diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb index b1cd46ac4..33d0cd6ad 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb @@ -85,6 +85,411 @@ module SchemaDefinition expect(proto_type_def_from(proto, "Account")).to include("Status status = 2;") expect(proto_type_def_from(proto, "User")).to include("Status status = 2;") end + it "raises on field-number mapping collisions for a message" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Account" => { + "id" => 1, + "name" => 1 + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("field-number mapping collision")) + end + + it "normalizes nil field-number mappings to empty hashes" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings nil + end + + expect(results.proto_schema).to eq("") + expect(results.proto_field_number_mappings).to eq({"enums" => {}, "messages" => {}}) + end + + it "validates field-number mapping input type" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings("bad") + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be a Hash")) + end + + it "validates that `messages` is a hash in field-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => "bad"}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must have a `messages` Hash")) + end + + it "accepts symbol `:messages` key in field-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + messages: { + "Account" => { + "id" => 7 + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 7;") + end + + it "accepts symbol `:fields` and nested symbol keys in field-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + messages: { + "Account" => { + fields: { + "id" => { + field_number: 7, + name_in_index: :account_id + } + } + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID", name_in_index: "account_id" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 7;") + end + + it "validates per-message field-number mapping structure" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => "bad"}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be a Hash")) + end + + it "validates that nested `fields` is a hash in field-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => {"fields" => "bad"}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must contain a `fields` Hash")) + end + + it "validates that mapped field numbers are valid protobuf field tags" do + [0, Schema::MAX_FIELD_NUMBER + 1, 19_000, 19_999].each do |invalid_number| + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => {"id" => invalid_number}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including( + "must be a valid protobuf field number", "excluding the reserved 19000-19999 range", invalid_number.to_s + )) + end + end + + it "accepts the maximum protobuf field number while allowing enum values in the field-reserved range" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => {"Account" => {"id" => Schema::MAX_FIELD_NUMBER}}, + # Enum value numbers have no protobuf-reserved range, so 19000-19999 is fine here. + "enums" => {"Status" => {"values" => {"ACTIVE" => 19_005}}} + } + ) + + s.enum_type "Status" do |t| + t.values "ACTIVE" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "accounts" + end + end + + generated = results.proto_schema + expect(generated).to include("string id = #{Schema::MAX_FIELD_NUMBER};") + expect(generated).to include("STATUS_ACTIVE = 19005;") + end + + it "skips the protobuf-reserved range when allocating new field numbers" do + legacy_mappings = (1..18_999).to_h { |number| ["legacy_field_#{number}", number] } + + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => legacy_mappings}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 20000;") + end + + it "validates that mapped enum value numbers do not exceed the int32 maximum" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"enums" => {"Status" => {"values" => {"ACTIVE" => Schema::MAX_ENUM_VALUE_NUMBER + 1}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be a positive integer no greater than 2147483647")) + end + + it "validates that mapped field numbers are integers" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => {"id" => "abc"}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be an integer")) + end + + it "accepts integer field numbers given as strings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => {"id" => "7"}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 7;") + end + + it "accepts symbol `:enums` and `:values` keys and shorthand value hashes in enum value-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + enums: { + "Status" => {values: {"INACTIVE" => 5}}, + "Color" => {"RED" => 3} + } + } + ) + + s.enum_type "Status" do |t| + t.values "ACTIVE", "INACTIVE" + end + + s.enum_type "Color" do |t| + t.values "RED" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.field "color", "Color" + t.index "accounts" + end + end + + generated = results.proto_schema + expect(generated).to include("STATUS_ACTIVE = 1;", "STATUS_INACTIVE = 5;", "COLOR_RED = 3;") + end + + it "validates that `enums` is a hash in enum value-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"enums" => "bad"}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must have an `enums` Hash")) + end + + it "validates per-enum value-number mapping structure" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"enums" => {"Status" => "bad"}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must contain a `values` Hash")) + end + + it "validates that mapped enum value numbers are positive integers, since 0 is reserved" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"enums" => {"Status" => {"values" => {"ACTIVE" => 0}}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be a positive integer", "_UNSPECIFIED")) + end + + it "validates that mapped enum value numbers are integers rather than truncating them" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"enums" => {"Status" => {"values" => {"ACTIVE" => 1.5}}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be a positive integer", "1.5")) + end + + it "validates that structured mappings include `field_number`" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"messages" => {"Account" => {"fields" => {"id" => {"name_in_index" => "account_id"}}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID", name_in_index: "account_id" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must include `field_number`")) + end + + it "validates that structured mappings use a String or Symbol `name_in_index`" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"messages" => {"Account" => {"fields" => {"id" => {"field_number" => 7, "name_in_index" => 123}}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must use a String or Symbol `name_in_index`")) + end + + it "defaults structured mappings without `name_in_index` to the field name" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"messages" => {"Account" => {"fields" => {"display_name" => {"field_number" => 7}}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "display_name", "String" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string display_name = 7;") + end + + it "allocates the next available field number when a renamed field has no old mapping entry" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"messages" => {"Account" => {"fields" => {"other_name" => 7}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "display_name", "String" do |f| + f.renamed_from "full_name" + end + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 1;") + expect(results.proto_schema).to include("string display_name = 2;") + end end end end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb index 4206565d5..d00198e2c 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb @@ -194,8 +194,115 @@ module SchemaDefinition expect(proto_type_def_from(proto, "Event")).to include("int64 occurred_at = 2;") end - it "uses public field names in schema.proto when `name_in_index` differs" do - proto = define_proto_schema do |s| + it "can assign field numbers from configured mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Account" => { + "id" => 10, + "name" => 2 + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "accounts" + end + end + + generated = results.proto_schema + expect(generated).to include("string id = 10;") + expect(generated).to include("string name = 2;") + end + + it "assigns new field numbers after mapped values when mappings are partial" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Account" => { + "id" => 1 + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string name = 2;") + end + + it "exposes generated field-number mappings as an artifact hash" do + results = define_proto_schema_results do |s| + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "accounts" + end + end + + expect(results.proto_field_number_mappings).to eq({ + "enums" => {}, + "messages" => { + "Account" => { + "fields" => { + "id" => 1, + "name" => 2 + } + } + } + }) + end + + it "preserves reserved numbers for removed fields and allocates new numbers above them" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Account" => { + "id" => 1, + "legacyField" => 2 + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "accounts" + end + end + + generated = results.proto_schema + expect(generated).to include("string id = 1;") + expect(generated).to include("string name = 3;") + + expect(results.proto_field_number_mappings).to eq({ + "enums" => {}, + "messages" => { + "Account" => { + "fields" => { + "id" => 1, + "legacyField" => 2, + "name" => 3 + } + } + } + }) + end + + it "uses public field names in schema.proto and stores name_in_index overrides in the mapping artifact" do + results = define_proto_schema_results do |s| s.object_type "Widget" do |t| t.field "id", "ID" t.field "display_name", "String", name_in_index: "displayName" @@ -203,9 +310,224 @@ module SchemaDefinition end end - widget = proto_type_def_from(proto, "Widget") - expect(widget).to include("string display_name = 2;") - expect(widget).not_to include("displayName") + expect(results.proto_schema).to include("string display_name = 2;") + expect(results.proto_schema).not_to include("displayName") + + expect(results.proto_field_number_mappings).to eq({ + "enums" => {}, + "messages" => { + "Widget" => { + "fields" => { + "id" => 1, + "display_name" => { + "field_number" => 2, + "name_in_index" => "displayName" + } + } + } + } + }) + end + + it "preserves a field number across a public field rename" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Account" => { + "fields" => { + "full_name" => 7 + } + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "display_name", "String" do |f| + f.renamed_from "full_name" + end + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 1;") + expect(results.proto_schema).to include("string display_name = 7;") + expect(results.proto_field_number_mappings).to eq({ + "enums" => {}, + "messages" => { + "Account" => { + "fields" => { + "id" => 1, + "display_name" => 7 + } + } + } + }) + end + + it "generates the proto schema and assigns field numbers when no renames are declared" do + results = define_proto_schema_results do |s| + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "display_name", "String" + t.index "accounts" + end + end + + expect(results.proto_schema).to eq(<<~PROTO) + syntax = "proto3"; + + package elasticgraph; + + message Account { + string id = 1; + string display_name = 2; + } + PROTO + + expect(results.proto_field_number_mappings).to eq({ + "enums" => {}, + "messages" => { + "Account" => { + "fields" => { + "id" => 1, + "display_name" => 2 + } + } + } + }) + end + + it "preserves enum value numbers from configured mappings, reserving removed values' numbers" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "enums" => { + "Status" => { + "values" => { + "ACTIVE" => 1, + "PAUSED" => 2, + "INACTIVE" => 3 + } + } + } + } + ) + + # `PAUSED` has been removed since the mappings were dumped, and `ARCHIVED` has been added. + s.enum_type "Status" do |t| + t.values "ACTIVE", "INACTIVE", "ARCHIVED" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "accounts" + end + end + + expect(proto_type_def_from(results.proto_schema, "Status")).to include( + "STATUS_ACTIVE = 1;", + "STATUS_INACTIVE = 3;", + "STATUS_ARCHIVED = 4;" + ) + + # `PAUSED` keeps its number reserved in the artifact so it is never reused for a new value. + expect(results.proto_field_number_mappings.fetch("enums")).to eq({ + "Status" => { + "values" => { + "ACTIVE" => 1, + "PAUSED" => 2, + "INACTIVE" => 3, + "ARCHIVED" => 4 + } + } + }) + end + + it "raises a clear error when the configured mappings assign the same number to two enum values" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"enums" => {"Status" => {"values" => {"ACTIVE" => 1, "INACTIVE" => 1}}}} + ) + + s.enum_type "Status" do |t| + t.values "ACTIVE", "INACTIVE" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including( + "enum value-number mapping collision in enum `Status`", + "`ACTIVE` and `INACTIVE`", + "number 1" + )) + end + + it "rejects field numbers that are not integers rather than silently truncating them" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"messages" => {"Account" => {"fields" => {"id" => 1.5}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("`Account.id`", "must be an integer", "1.5")) + end + + it "preserves stable field numbers for oneof alternatives" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Vehicle" => { + "fields" => { + "removed" => 1, + "car" => 4, + "bike" => 8 + } + } + } + } + ) + + ["Car", "Bike", "Scooter"].each do |type_name| + s.object_type type_name do |t| + t.field "id", "ID" + end + end + + s.union_type "Vehicle" do |t| + t.subtypes "Car", "Bike", "Scooter" + t.index "vehicles" + end + end + + vehicle = proto_type_def_from(results.proto_schema, "Vehicle") + expect(vehicle).to include("Car car = 4;", "Bike bike = 8;", "Scooter scooter = 2;") + expect(results.proto_field_number_mappings.fetch("messages").fetch("Vehicle")).to eq({ + "fields" => { + "removed" => 1, + "scooter" => 2, + "car" => 4, + "bike" => 8 + } + }) end it "renders independently each time it is called" do From 8803f5e07ecc830d6f44a735ca4bf35c03546f5a Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 16 Jul 2026 09:17:13 -0500 Subject: [PATCH 3/7] Cover protobuf numbering boundaries --- .../schema_edge_cases_spec.rb | 12 ++++++---- .../schema_definition/schema_spec.rb | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb index 33d0cd6ad..320cf2479 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb @@ -245,18 +245,21 @@ module SchemaDefinition end end - it "accepts the maximum protobuf field number while allowing enum values in the field-reserved range" do + it "accepts maximum protobuf numbers while allowing enum values in the field-reserved range" do results = define_proto_schema_results do |s| s.configure_proto_field_number_mappings( { "messages" => {"Account" => {"id" => Schema::MAX_FIELD_NUMBER}}, # Enum value numbers have no protobuf-reserved range, so 19000-19999 is fine here. - "enums" => {"Status" => {"values" => {"ACTIVE" => 19_005}}} + "enums" => {"Status" => {"values" => { + "ACTIVE" => Schema::MAX_ENUM_VALUE_NUMBER, + "INACTIVE" => 19_005 + }}} } ) s.enum_type "Status" do |t| - t.values "ACTIVE" + t.values "ACTIVE", "INACTIVE" end s.object_type "Account" do |t| @@ -268,7 +271,8 @@ module SchemaDefinition generated = results.proto_schema expect(generated).to include("string id = #{Schema::MAX_FIELD_NUMBER};") - expect(generated).to include("STATUS_ACTIVE = 19005;") + expect(generated).to include("STATUS_ACTIVE = #{Schema::MAX_ENUM_VALUE_NUMBER};") + expect(generated).to include("STATUS_INACTIVE = 19005;") end it "skips the protobuf-reserved range when allocating new field numbers" do diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb index d00198e2c..bf803b6b1 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb @@ -263,6 +263,29 @@ module SchemaDefinition }) end + it "exposes field and enum number mappings in canonical order" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "ZMessage" => {"fields" => {"second" => 2, "first" => 1}}, + "AMessage" => {"fields" => {"only" => 3}} + }, + "enums" => { + "ZEnum" => {"values" => {"SECOND" => 2, "FIRST" => 1}}, + "AEnum" => {"values" => {"ONLY" => 3}} + } + } + ) + end + + mappings = results.proto_field_number_mappings + expect(mappings.fetch("messages").keys).to eq(["AMessage", "ZMessage"]) + expect(mappings.dig("messages", "ZMessage", "fields").keys).to eq(["first", "second"]) + expect(mappings.fetch("enums").keys).to eq(["AEnum", "ZEnum"]) + expect(mappings.dig("enums", "ZEnum", "values").keys).to eq(["FIRST", "SECOND"]) + end + it "preserves reserved numbers for removed fields and allocates new numbers above them" do results = define_proto_schema_results do |s| s.configure_proto_field_number_mappings( From bf8fd074a6ddfc0daec275c29da12f4f526a2376 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 9 Jul 2026 10:52:21 -0500 Subject: [PATCH 4/7] Map DateTime to google.protobuf.Timestamp and document temporal string formats `DateTime` fields now use the well-known `google.protobuf.Timestamp` type (with `google/protobuf/timestamp.proto` imported automatically) instead of `string`. A Timestamp cannot be malformed the way a string can, and proto consumers get language-native timestamp types. Note that a Timestamp is a UTC instant, so a publisher's original UTC offset is not preserved; `t.protobuf type: "string"` remains available to override. To support this, `t.protobuf` gains two options usable by any scalar: - `import:` maps a scalar to an externally defined proto type, emitting the needed `import` statement in `schema.proto`. - `comment:` documents the expected format on each generated field, used by the built-in `string`-typed temporal scalars (`Date`, `LocalTime`, `TimeZone`) whose proto type is wider than the ElasticGraph type (e.g. `// ISO 8601 date`). Values are still validated at ingestion time, just as with JSON ingestion. --- elasticgraph-proto_ingestion/README.md | 53 ++++++++++---- .../schema_definition/factory_extension.rb | 38 +++++----- .../schema_definition/schema.rb | 9 +++ .../object_interface_and_union_extension.rb | 10 ++- .../schema_elements/scalar_type_extension.rb | 16 ++++- .../schema_definition/factory_extension.rbs | 2 +- .../schema_definition/schema.rbs | 1 + .../object_interface_and_union_extension.rbs | 2 +- .../schema_elements/scalar_type_extension.rbs | 4 +- .../schema_definition/api_extension_spec.rb | 1 + .../schema_definition/schema_spec.rb | 71 +++++++++++++++++++ 11 files changed, 167 insertions(+), 40 deletions(-) diff --git a/elasticgraph-proto_ingestion/README.md b/elasticgraph-proto_ingestion/README.md index fafefb9ff..d3379d44b 100644 --- a/elasticgraph-proto_ingestion/README.md +++ b/elasticgraph-proto_ingestion/README.md @@ -96,6 +96,22 @@ ElasticGraph.define_schema do |schema| end ``` +A custom scalar can also map to an externally defined proto type by passing `import:` with the +proto file that defines it, and `comment:` documents the expected format on each generated field +(useful when the proto type is wider than the ElasticGraph type): + +```ruby +# in config/schema/phone_number.rb + +ElasticGraph.define_schema do |schema| + schema.scalar_type "PhoneNumber" do |t| + t.mapping type: "keyword" + t.json_schema type: "string" + t.protobuf type: "string", comment: "E.164 phone number" + end +end +``` + ### Stable Field Numbers `schema_artifacts:dump` automatically reads and writes `proto_field_numbers.yaml` @@ -143,23 +159,30 @@ enums: The generated `schema.proto` uses these built-in scalar mappings: -| ElasticGraph Type | Protobuf Type | -|-------------------|------------| -| `Boolean` | `bool` | -| `Cursor` | `string` | -| `Date` | `string` | -| `DateTime` | `string` | -| `Float` | `double` | -| `ID` | `string` | -| `Int` | `int32` | -| `JsonSafeLong` | `int64` | -| `LocalTime` | `string` | -| `LongString` | `int64` | -| `String` | `string` | -| `TimeZone` | `string` | -| `Untyped` | `string` | +| ElasticGraph Type | Protobuf Type | +|-------------------|-----------------------------| +| `Boolean` | `bool` | +| `Cursor` | `string` | +| `Date` | `string` | +| `DateTime` | `google.protobuf.Timestamp` | +| `Float` | `double` | +| `ID` | `string` | +| `Int` | `int32` | +| `JsonSafeLong` | `int64` | +| `LocalTime` | `string` | +| `LongString` | `int64` | +| `String` | `string` | +| `TimeZone` | `string` | +| `Untyped` | `string` | Additionally: +- `DateTime` uses the [well-known `Timestamp` type](https://protobuf.dev/reference/protobuf/google.protobuf/#timestamp); + `schema.proto` imports `google/protobuf/timestamp.proto` automatically. Note that a `Timestamp` + is a UTC instant, so a publisher's original UTC offset is not preserved. +- `string`-typed temporal scalars (`Date`, `LocalTime`, `TimeZone`) are wider than the + ElasticGraph types they carry, so generated fields of these types document the expected format + in a comment (e.g. `// ISO 8601 date, e.g. "2024-11-25"`). Values are validated when events + are ingested, just as with JSON ingestion. - List types become `repeated` fields. - Lists of lists (e.g. `[[Float!]!]!`) are not supported because Protocol Buffers cannot represent them directly. Schema artifact generation raises an error identifying the unsupported field. diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb index e81f09a8f..df7fb9da7 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb @@ -18,21 +18,21 @@ module ProtoIngestion module SchemaDefinition # Extension module applied to Factory to add proto support. module FactoryExtension - # Default protobuf types applied to ElasticGraph's built-in scalar types as they are constructed. - BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME = { - "Boolean" => "bool", - "Cursor" => "string", - "Date" => "string", - "DateTime" => "string", - "Float" => "double", - "ID" => "string", - "Int" => "int32", - "JsonSafeLong" => "int64", - "LocalTime" => "string", - "LongString" => "int64", - "String" => "string", - "TimeZone" => "string", - "Untyped" => "string" + # Default protobuf options applied to ElasticGraph's built-in scalar types as they are constructed. + BUILT_IN_SCALAR_PROTO_OPTIONS_BY_NAME = { + "Boolean" => {type: "bool"}, + "Cursor" => {type: "string"}, + "Date" => {type: "string", comment: %(ISO 8601 date, e.g. "2024-11-25")}, + "DateTime" => {type: "google.protobuf.Timestamp", import: "google/protobuf/timestamp.proto"}, + "Float" => {type: "double"}, + "ID" => {type: "string"}, + "Int" => {type: "int32"}, + "JsonSafeLong" => {type: "int64"}, + "LocalTime" => {type: "string", comment: %(ISO 8601 local time, e.g. "14:23:12")}, + "LongString" => {type: "int64"}, + "String" => {type: "string"}, + "TimeZone" => {type: "string", comment: %(IANA time zone identifier, e.g. "America/Los_Angeles")}, + "Untyped" => {type: "string"} }.freeze # Creates a new enum type with proto extensions. @@ -97,8 +97,12 @@ def new_scalar_type(name) extended_type = type.extend(SchemaElements::ScalarTypeExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension if state.initially_registered_built_in_types.empty? && - (proto_type = BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME[name.to_s]) - extended_type.protobuf type: proto_type + (proto_options = BUILT_IN_SCALAR_PROTO_OPTIONS_BY_NAME[name.to_s]) + extended_type.protobuf( + type: proto_options.fetch(:type), + import: proto_options[:import], + comment: proto_options[:comment] + ) end yield extended_type if block_given? diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb index 5b43274f4..03d1a41df 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb @@ -61,6 +61,7 @@ def to_proto sections = [ %(syntax = "proto3";), "package #{@package_name};", + *render_imports(types), render_definitions(types) ] @@ -177,6 +178,14 @@ def render_definitions(types) .join("\n\n") end + def render_imports(types) + imports = types.filter_map do |type| + type.protobuf_import if type.respond_to?(:protobuf_import) + end.uniq.sort + + imports.empty? ? [] : [imports.map { |import| %(import "#{import}";) }.join("\n")] + end + def valid_field_number?(number) number.between?(1, MAX_FIELD_NUMBER) && !RESERVED_FIELD_NUMBER_RANGE.cover?(number) end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb index 1f3ab6e44..a8d11dd3b 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb @@ -68,7 +68,7 @@ def render_proto_message(schema, message_name) lines << "message #{message_name} {" fields.each do |schema_field, field| field_name = Identifier.field_name(field.name) - repeated, field_type = proto_field_type_for(field.type, context_field_name: field.name) + repeated, field_type, type_comment = proto_field_type_for(field.type, context_field_name: field.name) field_number = schema.field_number_for( message_name: message_name, type_name: name, @@ -77,7 +77,10 @@ def render_proto_message(schema, message_name) ) label = "repeated " if repeated line = " #{label}#{field_type} #{field_name} = #{field_number};" - line += " // source name: #{field.name}" if field_name != field.name + comment_parts = [] # : ::Array[::String] + comment_parts << "source name: #{field.name}" if field_name != field.name + comment_parts << type_comment if type_comment + line += " // #{comment_parts.join("; ")}" if comment_parts.any? lines.concat(ProtoDocumentation.comment_lines_for(schema_field.doc_comment, indent: " ")) lines << line end @@ -126,7 +129,8 @@ def proto_field_type_for(type_ref, context_field_name:) end proto_type = _ = base_type_ref.resolved - [list_depth == 1, proto_type.proto_name] + type_comment = (ScalarTypeExtension === proto_type) ? proto_type.protobuf_comment : nil + [list_depth == 1, proto_type.proto_name, type_comment] end def list_depth_and_base_type(type_ref) diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb index 5a8c724a3..5454f586a 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb @@ -18,12 +18,24 @@ module ScalarTypeExtension # @dynamic protobuf_type attr_reader :protobuf_type + # Proto file to import for the configured protobuf type, if it is externally defined. + # @dynamic protobuf_import + attr_reader :protobuf_import + + # Comment rendered on generated proto fields of this scalar type. + # @dynamic protobuf_comment + attr_reader :protobuf_comment + # Configures the protobuf type for this scalar type. # - # @param type [String] protobuf scalar type name + # @param type [String] protobuf type name + # @param import [String, nil] proto file to import for an externally defined type + # @param comment [String, nil] comment rendered on generated fields of this type # @return [void] - def protobuf(type:) + def protobuf(type:, import: nil, comment: nil) @protobuf_type = type + @protobuf_import = import + @protobuf_comment = comment @proto_name = nil end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/factory_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/factory_extension.rbs index 4822f3fb3..39a29dedc 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/factory_extension.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/factory_extension.rbs @@ -2,7 +2,7 @@ module ElasticGraph module ProtoIngestion module SchemaDefinition module FactoryExtension: ::ElasticGraph::SchemaDefinition::Factory - BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME: ::Hash[::String, ::String] + BUILT_IN_SCALAR_PROTO_OPTIONS_BY_NAME: ::Hash[::String, ::Hash[::Symbol, ::String]] def new_enum_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumType & SchemaElements::EnumTypeExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::EnumType & SchemaElements::EnumTypeExtension) def new_enum_value: (::String, ::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & SchemaElements::EnumValueExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & SchemaElements::EnumValueExtension) diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs index 7e7f15f70..5377adf08 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs @@ -45,6 +45,7 @@ module ElasticGraph def proto_types: () -> ::Array[untyped] def render_definitions: (::Array[untyped] types) -> ::String + def render_imports: (::Array[untyped] types) -> ::Array[::String] def valid_field_number?: (::Integer) -> bool def next_available_field_number: (::Set[::Integer]) -> ::Integer def next_available_enum_value_number: (::Set[::Integer]) -> ::Integer diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs index b1d562c7f..2ef470ad7 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs @@ -21,7 +21,7 @@ module ElasticGraph def proto_field_type_for: ( ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference, context_field_name: ::String - ) -> [bool, ::String] + ) -> [bool, ::String, ::String?] def list_depth_and_base_type: ( ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference ) -> [::Integer, ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference] diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs index 817d6fee7..89ac66542 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs @@ -6,8 +6,10 @@ module ElasticGraph @proto_name: ::String? attr_reader protobuf_type: ::String? + attr_reader protobuf_import: ::String? + attr_reader protobuf_comment: ::String? - def protobuf: (type: ::String) -> void + def protobuf: (type: ::String, ?import: ::String?, ?comment: ::String?) -> void def finalize_protobuf_configuration!: () -> void def proto_name: () -> ::String def to_proto: () -> nil diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb index 23383ab54..c0a107d9f 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb @@ -38,6 +38,7 @@ module SchemaDefinition end expect(field_types).to match_array(FactoryExtension::BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME.keys) + expect(proto).to include('import "google/protobuf/timestamp.proto";') FactoryExtension::BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME.each.with_index(1) do |(type_name, proto_type), field_number| field_name = Identifier.field_name(type_name.downcase) expect(proto).to include("#{proto_type} #{field_name} = #{field_number};") diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb index bf803b6b1..3ba45f6b0 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb @@ -194,6 +194,77 @@ module SchemaDefinition expect(proto_type_def_from(proto, "Event")).to include("int64 occurred_at = 2;") end + it "maps `DateTime` fields to `google.protobuf.Timestamp`, importing its proto file once" do + proto = define_proto_schema do |s| + s.object_type "Event" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + t.field "updated_at", "DateTime" + t.index "events" + end + end + + expect(proto).to eq(<<~PROTO) + syntax = "proto3"; + + package elasticgraph; + + import "google/protobuf/timestamp.proto"; + + message Event { + string id = 1; + google.protobuf.Timestamp created_at = 2; + google.protobuf.Timestamp updated_at = 3; + } + PROTO + end + + it "renders format comments on fields whose scalar type documents one" do + proto = define_proto_schema do |s| + s.object_type "Person" do |t| + t.field "id", "ID" + t.field "birth_date", "Date" + t.index "people" + end + end + + expect(proto).to include( + %(string birth_date = 2; // ISO 8601 date, e.g. "2024-11-25") + ) + end + + it "combines source name and format comments on a single field" do + proto = define_proto_schema do |s| + s.object_type "Person" do |t| + t.field "id", "ID" + t.field "option", "Date" + t.index "people" + end + end + + expect(proto).to include( + %(string option_ = 2; // source name: option; ISO 8601 date, e.g. "2024-11-25") + ) + end + + it "supports `import:` and `comment:` on custom scalar types" do + proto = define_proto_schema do |s| + s.scalar_type "Money" do |t| + t.mapping type: "keyword" + t.protobuf type: "myapp.types.Money", import: "myapp/types/money.proto", comment: "amount + currency" + end + + s.object_type "Order" do |t| + t.field "id", "ID" + t.field "total", "Money" + t.index "orders" + end + end + + expect(proto).to include('import "myapp/types/money.proto";') + expect(proto).to include("myapp.types.Money total = 2; // amount + currency") + end + it "can assign field numbers from configured mappings" do results = define_proto_schema_results do |s| s.configure_proto_field_number_mappings( From b94fd0e95dd0a17df1192ed7b2c425f3591a7515 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 16 Jul 2026 09:17:48 -0500 Subject: [PATCH 5/7] Test protobuf import stability --- .../schema_definition/api_extension_spec.rb | 9 +++--- .../schema_definition/schema_spec.rb | 32 +++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb index c0a107d9f..5a2dba5d2 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb @@ -26,10 +26,11 @@ module SchemaDefinition end it "maps every built-in scalar to a proto field type" do + built_in_scalar_options = FactoryExtension::BUILT_IN_SCALAR_PROTO_OPTIONS_BY_NAME field_types = [] proto = define_proto_schema do |s| s.object_type "Widget" do |t| - FactoryExtension::BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME.each_key do |type_name| + built_in_scalar_options.each_key do |type_name| t.field type_name.downcase, type_name end field_types = t.graphql_fields_by_name.values.map { |field| field.type.name } @@ -37,11 +38,11 @@ module SchemaDefinition end end - expect(field_types).to match_array(FactoryExtension::BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME.keys) + expect(field_types).to match_array(built_in_scalar_options.keys) expect(proto).to include('import "google/protobuf/timestamp.proto";') - FactoryExtension::BUILT_IN_SCALAR_PROTO_TYPES_BY_NAME.each.with_index(1) do |(type_name, proto_type), field_number| + built_in_scalar_options.each.with_index(1) do |(type_name, options), field_number| field_name = Identifier.field_name(type_name.downcase) - expect(proto).to include("#{proto_type} #{field_name} = #{field_number};") + expect(proto).to include("#{options.fetch(:type)} #{field_name} = #{field_number};") end end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb index 3ba45f6b0..b730ff516 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb @@ -219,6 +219,38 @@ module SchemaDefinition PROTO end + it "sorts and de-duplicates imports shared by distinct protobuf types" do + proto = define_proto_schema do |s| + s.scalar_type "FirstZType" do |t| + t.mapping type: "keyword" + t.protobuf type: "example.FirstZType", import: "z/types.proto" + end + + s.scalar_type "SecondZType" do |t| + t.mapping type: "keyword" + t.protobuf type: "example.SecondZType", import: "z/types.proto" + end + + s.scalar_type "AType" do |t| + t.mapping type: "keyword" + t.protobuf type: "example.AType", import: "a/types.proto" + end + + s.object_type "Event" do |t| + t.field "id", "ID" + t.field "first_z", "FirstZType" + t.field "second_z", "SecondZType" + t.field "a", "AType" + t.index "events" + end + end + + expect(proto.lines.grep(/\Aimport /).map(&:chomp)).to eq([ + %(import "a/types.proto";), + %(import "z/types.proto";) + ]) + end + it "renders format comments on fields whose scalar type documents one" do proto = define_proto_schema do |s| s.object_type "Person" do |t| From bac4cdce9794a55d3930f4fe65188feebca0e2a2 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 8 Jul 2026 18:19:47 -0500 Subject: [PATCH 6/7] Support proto2 syntax and custom proto file headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `proto_schema_artifacts` gains two options: - `syntax: :proto2` emits a proto2 file instead of the default proto3 (labeling every field `optional` or `repeated`). This is useful when the generated messages need to reference proto2 types — `protoc` forbids a proto3 message from referencing a proto2 enum. - `headers:` injects file-level lines (such as `option` declarations) verbatim as a contiguous section after the `package` declaration, so language-specific options can be set without baking any particular convention into the gem. --- elasticgraph-proto_ingestion/README.md | 52 ++++++++++++++++++- .../schema_definition/api_extension.rb | 15 +++++- .../proto_ingestion_state.rb | 2 +- .../schema_definition/results_extension.rb | 4 +- .../schema_definition/schema.rb | 25 +++++++-- .../object_interface_and_union_extension.rb | 2 +- .../schema_definition/state_extension.rb | 7 ++- .../schema_definition/api_extension.rbs | 6 ++- .../proto_ingestion_state.rbs | 9 +++- .../schema_definition/schema.rbs | 9 +++- .../schema_definition/api_extension_spec.rb | 25 +++++++++ .../schema_definition/schema_spec.rb | 50 ++++++++++++++++++ 12 files changed, 193 insertions(+), 13 deletions(-) diff --git a/elasticgraph-proto_ingestion/README.md b/elasticgraph-proto_ingestion/README.md index d3379d44b..2e1c3dce1 100644 --- a/elasticgraph-proto_ingestion/README.md +++ b/elasticgraph-proto_ingestion/README.md @@ -1,7 +1,9 @@ # ElasticGraph::ProtoIngestion An ElasticGraph extension that supports ingesting Protocol Buffer data into ElasticGraph. -Currently it generates `proto3` Protocol Buffers schema artifacts from ElasticGraph schemas. +Currently it generates Protocol Buffers schema artifacts from ElasticGraph schemas: it emits +`proto3` by default and can emit `proto2`, and supports arbitrary file-level headers (such +as `option` declarations). ## Dependency Diagram @@ -79,6 +81,54 @@ After running `bundle exec rake schema_artifacts:dump`, ElasticGraph will genera ## Schema Definition API +### Protobuf Syntax (`proto2` / `proto3`) + +`proto_schema_artifacts` emits `proto3` by default. Pass `syntax: :proto2` to emit a proto2 file +instead (every field is then labeled `optional` or `repeated`). This is useful when the generated +messages need to reference proto2 types — for example, `protoc` forbids a `proto3` message from +referencing a `proto2` enum: + +```ruby +# in config/schema/protobuf.rb + +ElasticGraph.define_schema do |schema| + schema.proto_schema_artifacts package_name: "myapp.events.v1", syntax: :proto2 +end +``` + +### Custom Headers + +Pass `headers:` an array of strings to inject file-level lines (such as `option` declarations) +verbatim, as a contiguous section immediately after the `package` declaration. This lets you set +language-specific options without the gem baking in any particular convention: + +```ruby +# in config/schema/protobuf.rb + +ElasticGraph.define_schema do |schema| + schema.proto_schema_artifacts( + package_name: "myapp.events.v1", + headers: [ + %(option java_package = "com.myapp.events";), + "option java_multiple_files = true;" + ] + ) +end +``` + +produces: + +```text +syntax = "proto3"; + +package myapp.events.v1; + +option java_package = "com.myapp.events"; +option java_multiple_files = true; + +// ...messages... +``` + ### Custom Scalar Types Built-in ElasticGraph scalar types are automatically mapped to proto scalar types. diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb index fa692a5ec..065d855d7 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb @@ -35,18 +35,29 @@ def self.extended(api) # Configures protobuf artifact generation behavior. # # @param package_name [String] proto package name to emit + # @param syntax [Symbol] `:proto3` (default) or `:proto2` + # @param headers [Array] file-level lines rendered verbatim after the package declaration # @return [void] # # @example Set the proto package name # ElasticGraph.define_schema do |schema| # schema.proto_schema_artifacts package_name: "myapp.events.v1" # end - def proto_schema_artifacts(package_name:) + def proto_schema_artifacts(package_name:, syntax: :proto3, headers: []) if !package_name.is_a?(String) || package_name.empty? raise Errors::SchemaError, "`package_name` must be a non-empty String" end + unless Schema::SUPPORTED_SYNTAXES.include?(syntax.to_s) + raise Errors::SchemaError, "`syntax` must be one of #{Schema::SUPPORTED_SYNTAXES.inspect}, got: #{syntax.inspect}" + end + if !headers.is_a?(Array) || headers.any? { |header| !header.is_a?(String) } + raise Errors::SchemaError, "`headers` must be an Array of Strings" + end - proto_ingestion_state.package_name = Identifier.validate_package_name(package_name) + ingestion_state = proto_ingestion_state + ingestion_state.package_name = Identifier.validate_package_name(package_name) + ingestion_state.syntax = syntax + ingestion_state.headers = headers nil end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb index e5c4d921a..248eae6ad 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb @@ -14,7 +14,7 @@ module SchemaDefinition # Holds the proto ingestion extension's schema definition state. # # @private - class ProtoIngestionState < ::Struct.new(:package_name, :field_number_mappings) + class ProtoIngestionState < ::Struct.new(:package_name, :field_number_mappings, :syntax, :headers) def initialize(...) super @type_name_by_proto_name = {} diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb index 6e9f3be3d..82909e64c 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb @@ -42,7 +42,9 @@ def protobuf_schema_generator state: extension_state, all_types: all_types, package_name: ingestion_state.package_name, - proto_field_number_mappings: ingestion_state.field_number_mappings + proto_field_number_mappings: ingestion_state.field_number_mappings, + syntax: ingestion_state.syntax, + headers: ingestion_state.headers ) end end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb index 03d1a41df..e89e5e685 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb @@ -15,7 +15,7 @@ module ElasticGraph module ProtoIngestion module SchemaDefinition - # Builds a `proto3` schema string from an ElasticGraph schema definition. + # Builds a `proto2` or `proto3` schema string from an ElasticGraph schema definition. class Schema # Internal representation of a stored field-number mapping. # @@ -31,6 +31,8 @@ class Schema RESERVED_FIELD_NUMBER_RANGE = 19_000..19_999 # The largest enum value number protobuf allows (the int32 maximum). MAX_ENUM_VALUE_NUMBER = 2_147_483_647 + # Protobuf syntaxes this generator can emit. + SUPPORTED_SYNTAXES = %w[proto2 proto3].freeze # @param state [ElasticGraph::SchemaDefinition::State] # @param all_types [Array] @@ -40,8 +42,12 @@ def initialize( state:, all_types:, package_name:, - proto_field_number_mappings: {} + proto_field_number_mappings: {}, + syntax: :proto3, + headers: [] ) + @syntax = syntax.to_s + @headers = headers @state = state @all_types = all_types @package_name = Identifier.validate_package_name(package_name) @@ -59,8 +65,9 @@ def to_proto return "" if types.empty? sections = [ - %(syntax = "proto3";), + %(syntax = "#{@syntax}";), "package #{@package_name};", + *render_headers, *render_imports(types), render_definitions(types) ] @@ -150,6 +157,14 @@ def enum_value_numbers_for(enum_name, value_names) end end + # Returns the label for a protobuf field under the configured syntax. + # + # @api private + def field_label(repeated) + return repeated ? "repeated " : "optional " if @syntax == "proto2" + repeated ? "repeated " : nil + end + private # Selects the indexed root types and every type transitively referenced by their protobuf @@ -186,6 +201,10 @@ def render_imports(types) imports.empty? ? [] : [imports.map { |import| %(import "#{import}";) }.join("\n")] end + def render_headers + @headers.empty? ? [] : [@headers.join("\n")] + end + def valid_field_number?(number) number.between?(1, MAX_FIELD_NUMBER) && !RESERVED_FIELD_NUMBER_RANGE.cover?(number) end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb index a8d11dd3b..6e983886c 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb @@ -75,7 +75,7 @@ def render_proto_message(schema, message_name) public_field_name: schema_field.name, name_in_index: field.name_in_index ) - label = "repeated " if repeated + label = schema.field_label(repeated) line = " #{label}#{field_type} #{field_name} = #{field_number};" comment_parts = [] # : ::Array[::String] comment_parts << "source name: #{field.name}" if field_name != field.name diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb index 891ae5a2d..c9ccef295 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb @@ -21,7 +21,12 @@ module StateExtension def self.extended(state) state.instance_variable_set( :@proto_ingestion_state, - ProtoIngestionState.new(package_name: "elasticgraph", field_number_mappings: {}) + ProtoIngestionState.new( + package_name: "elasticgraph", + field_number_mappings: {}, + syntax: :proto3, + headers: [] + ) ) end end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs index e8ac37ee5..5e340ecd5 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs @@ -3,7 +3,11 @@ module ElasticGraph module SchemaDefinition module APIExtension: ::ElasticGraph::SchemaDefinition::API def self.extended: (::ElasticGraph::SchemaDefinition::API & APIExtension) -> void - def proto_schema_artifacts: (package_name: ::String) -> void + def proto_schema_artifacts: ( + package_name: ::String, + ?syntax: (::Symbol | ::String), + ?headers: ::Array[::String] + ) -> void def configure_proto_field_number_mappings: (untyped) -> void private diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs index 5947f7c14..bf33fd64a 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs @@ -4,8 +4,15 @@ module ElasticGraph class ProtoIngestionStateSupertype attr_accessor package_name: ::String attr_accessor field_number_mappings: untyped + attr_accessor syntax: (::Symbol | ::String) + attr_accessor headers: ::Array[::String] - def initialize: (package_name: ::String, field_number_mappings: untyped) -> void + def initialize: ( + package_name: ::String, + field_number_mappings: untyped, + syntax: (::Symbol | ::String), + headers: ::Array[::String] + ) -> void end class ProtoIngestionState < ProtoIngestionStateSupertype diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs index 5377adf08..f786d1464 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs @@ -14,7 +14,10 @@ module ElasticGraph MAX_FIELD_NUMBER: ::Integer RESERVED_FIELD_NUMBER_RANGE: ::Range[::Integer] MAX_ENUM_VALUE_NUMBER: ::Integer + SUPPORTED_SYNTAXES: ::Array[::String] + @syntax: ::String + @headers: ::Array[::String] @state: ::ElasticGraph::SchemaDefinition::State @all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] @package_name: ::String @@ -28,7 +31,9 @@ module ElasticGraph state: ::ElasticGraph::SchemaDefinition::State, all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType], package_name: ::String, - ?proto_field_number_mappings: untyped + ?proto_field_number_mappings: untyped, + ?syntax: (::Symbol | ::String), + ?headers: ::Array[::String] ) -> void def to_proto: () -> ::String @@ -40,12 +45,14 @@ module ElasticGraph name_in_index: ::String ) -> ::Integer def enum_value_numbers_for: (::String, ::Array[::String]) -> ::Hash[::String, ::Integer] + def field_label: (bool) -> ::String? private def proto_types: () -> ::Array[untyped] def render_definitions: (::Array[untyped] types) -> ::String def render_imports: (::Array[untyped] types) -> ::Array[::String] + def render_headers: () -> ::Array[::String] def valid_field_number?: (::Integer) -> bool def next_available_field_number: (::Set[::Integer]) -> ::Integer def next_available_enum_value_number: (::Set[::Integer]) -> ::Integer diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb index 5a2dba5d2..e2b8733c8 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb @@ -60,6 +60,31 @@ module SchemaDefinition }.to raise_error(Errors::SchemaError, a_string_including("`package_name` must be a non-empty String")) end + it "requires `syntax` to be a supported protobuf syntax" do + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: "elasticgraph", syntax: :proto1 + end + }.to raise_error(Errors::SchemaError, a_string_including("`syntax` must be one of")) + end + + it "requires `headers` to be an Array of Strings" do + expect { + define_proto_schema do |s| + s.proto_schema_artifacts( + package_name: "elasticgraph", + headers: %(option java_package = "com.example";) + ) + end + }.to raise_error(Errors::SchemaError, a_string_including("`headers` must be an Array of Strings")) + + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: "elasticgraph", headers: [:not_a_string] + end + }.to raise_error(Errors::SchemaError, a_string_including("`headers` must be an Array of Strings")) + end + it "rejects invalid package names when they are configured" do expect { define_proto_schema do |s| diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb index b730ff516..610f91387 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb @@ -73,6 +73,56 @@ module SchemaDefinition PROTO end + it "emits proto2 syntax with an explicit label on every field when `syntax: :proto2`" do + proto = define_proto_schema do |s| + s.proto_schema_artifacts package_name: "elasticgraph", syntax: :proto2 + + s.enum_type "Status" do |t| + t.values "ACTIVE", "INACTIVE" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.field "tags", "[String!]!" + t.index "accounts" + end + end + + expect(proto).to start_with(%(syntax = "proto2";)) + expect(proto_type_def_from(proto, "Account")).to include( + "optional string id = 1;", + "optional Status status = 2;", + "repeated string tags = 3;" + ) + end + + it "renders custom `headers` verbatim after the package declaration" do + proto = define_proto_schema do |s| + s.proto_schema_artifacts( + package_name: "myapp.events.v1", + headers: [ + %(option java_package = "com.myapp.events";), + "option java_multiple_files = true;" + ] + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect(proto).to include(<<~PROTO) + package myapp.events.v1; + + option java_package = "com.myapp.events"; + option java_multiple_files = true; + + message Account { + PROTO + end + it "generates oneof wrappers for indexed interface and union types" do proto = define_proto_schema do |s| s.object_type "Car" do |t| From 9a7765f021fc3473db326f11b31376cb9b579c04 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 16 Jul 2026 09:18:25 -0500 Subject: [PATCH 7/7] Test mixed protobuf header validation --- .../proto_ingestion/schema_definition/api_extension_spec.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb index e2b8733c8..47981d30c 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb @@ -83,6 +83,12 @@ module SchemaDefinition s.proto_schema_artifacts package_name: "elasticgraph", headers: [:not_a_string] end }.to raise_error(Errors::SchemaError, a_string_including("`headers` must be an Array of Strings")) + + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: "elasticgraph", headers: ["valid", :not_a_string] + end + }.to raise_error(Errors::SchemaError, a_string_including("`headers` must be an Array of Strings")) end it "rejects invalid package names when they are configured" do