diff --git a/elasticgraph-proto_ingestion/README.md b/elasticgraph-proto_ingestion/README.md index ca9ad4bcb..2e1c3dce1 100644 --- a/elasticgraph-proto_ingestion/README.md +++ b/elasticgraph-proto_ingestion/README.md @@ -1,6 +1,9 @@ # ElasticGraph::ProtoIngestion An ElasticGraph extension that supports ingesting Protocol Buffer data into ElasticGraph. +Currently it generates Protocol Buffers schema artifacts from ElasticGraph schemas: it emits +`proto3` by default and can emit `proto2`, and supports arbitrary file-level headers (such +as `option` declarations). ## Dependency Diagram @@ -15,3 +18,222 @@ graph LR; elasticgraph-proto_ingestion --> elasticgraph-support; class elasticgraph-support otherEgGemStyle; ``` + +## Usage + +First, add `elasticgraph-proto_ingestion` to your `Gemfile`, alongside the other ElasticGraph gems: + +```diff +diff --git a/Gemfile b/Gemfile +index 4a5ef1e..5c16c2b 100644 +--- a/Gemfile ++++ b/Gemfile +@@ -8,6 +8,7 @@ gem "elasticgraph-query_registry", *elasticgraph_details + + # Can be elasticgraph-elasticsearch or elasticgraph-opensearch based on the datastore you want to use. + gem "elasticgraph-opensearch", *elasticgraph_details ++gem "elasticgraph-proto_ingestion", *elasticgraph_details + + gem "httpx", "~> 1.3" + +``` + +Next, update your `Rakefile` so that `ElasticGraph::ProtoIngestion::SchemaDefinition::APIExtension` is +included in the schema-definition extension modules: + +```diff +diff --git a/Rakefile b/Rakefile +index 2943335..26633c3 100644 +--- a/Rakefile ++++ b/Rakefile +@@ -3,5 +3,6 @@ + require "elastic_graph/json_ingestion/schema_definition/api_extension" + require "elastic_graph/local/rake_tasks" ++require "elastic_graph/proto_ingestion/schema_definition/api_extension" + require "elastic_graph/query_registry/rake_tasks" + require "rspec/core/rake_task" + require "standard/rake" +@@ -16,6 +17,7 @@ ElasticGraph::Local::RakeTasks.new( + # Determines casing of field names. Can be either `:camelCase` or `:snake_case`. + tasks.schema_element_name_form = :camelCase + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension ++ tasks.schema_definition_extension_modules << ElasticGraph::ProtoIngestion::SchemaDefinition::APIExtension + + # Customizes the names of fields generated by ElasticGraph. + tasks.schema_element_name_overrides = { +``` + +Adding the schema definition extension automatically enables `schema.proto` generation with the default +`elasticgraph` package. Optionally, configure a custom package name from your schema definition: + +```ruby +# in config/schema/protobuf.rb + +ElasticGraph.define_schema do |schema| + schema.proto_schema_artifacts package_name: "myapp.events.v1" +end +``` + +After running `bundle exec rake schema_artifacts:dump`, ElasticGraph will generate: + +- `schema.proto` +- `proto_field_numbers.yaml` + +## Schema Definition API + +### Protobuf Syntax (`proto2` / `proto3`) + +`proto_schema_artifacts` emits `proto3` by default. Pass `syntax: :proto2` to emit a proto2 file +instead (every field is then labeled `optional` or `repeated`). This is useful when the generated +messages need to reference proto2 types — for example, `protoc` forbids a `proto3` message from +referencing a `proto2` enum: + +```ruby +# in config/schema/protobuf.rb + +ElasticGraph.define_schema do |schema| + schema.proto_schema_artifacts package_name: "myapp.events.v1", syntax: :proto2 +end +``` + +### Custom Headers + +Pass `headers:` an array of strings to inject file-level lines (such as `option` declarations) +verbatim, as a contiguous section immediately after the `package` declaration. This lets you set +language-specific options without the gem baking in any particular convention: + +```ruby +# in config/schema/protobuf.rb + +ElasticGraph.define_schema do |schema| + schema.proto_schema_artifacts( + package_name: "myapp.events.v1", + headers: [ + %(option java_package = "com.myapp.events";), + "option java_multiple_files = true;" + ] + ) +end +``` + +produces: + +```text +syntax = "proto3"; + +package myapp.events.v1; + +option java_package = "com.myapp.events"; +option java_multiple_files = true; + +// ...messages... +``` + +### Custom Scalar Types + +Built-in ElasticGraph scalar types are automatically mapped to proto scalar types. +For custom scalar types, use `protobuf` to define the proto scalar type: + +```ruby +# in config/schema/email.rb + +ElasticGraph.define_schema do |schema| + schema.scalar_type "Email" do |t| + t.mapping type: "keyword" + t.json_schema type: "string", format: "email" + t.protobuf type: "string" + end +end +``` + +A custom scalar can also map to an externally defined proto type by passing `import:` with the +proto file that defines it, and `comment:` documents the expected format on each generated field +(useful when the proto type is wider than the ElasticGraph type): + +```ruby +# in config/schema/phone_number.rb + +ElasticGraph.define_schema do |schema| + schema.scalar_type "PhoneNumber" do |t| + t.mapping type: "keyword" + t.json_schema type: "string" + t.protobuf type: "string", comment: "E.164 phone number" + end +end +``` + +### Stable Field Numbers + +`schema_artifacts:dump` automatically reads and writes `proto_field_numbers.yaml` +in the schema artifacts directory. Existing numbers stay fixed even if field order +changes, and new fields get the next available numbers. Field numbers follow protobuf's +rules: they must be between 1 and 536,870,911, and the protobuf-reserved 19000-19999 +range is never allocated and is rejected in mappings. + +Alternatives inside generated interface and union `oneof` blocks use the same stable +message-field mappings, so adding or removing a concrete subtype does not renumber the +remaining alternatives. + +`schema.proto` always uses the public GraphQL field names. When a field uses a +different `name_in_index`, the sidecar YAML stores that override privately: + +```yaml +messages: + Widget: + fields: + id: 1 + display_name: + field_number: 2 + name_in_index: displayName +``` + +If a field is renamed with `field.renamed_from`, `elasticgraph-proto_ingestion` reuses the +existing field number under the new public field name. + +### Stable Enum Value Numbers + +Enum value numbers are pinned the same way, in an `enums` section of the sidecar. Existing +values keep their numbers when other values are added or removed, new values get the next +available numbers, and removed values keep their numbers reserved so they are never reused +(number `0` is always the generated `*_UNSPECIFIED` value): + +```yaml +enums: + WidgetColor: + values: + RED: 1 + BLUE: 2 +``` + +## Type Mappings + +The generated `schema.proto` uses these built-in scalar mappings: + +| ElasticGraph Type | Protobuf Type | +|-------------------|-----------------------------| +| `Boolean` | `bool` | +| `Cursor` | `string` | +| `Date` | `string` | +| `DateTime` | `google.protobuf.Timestamp` | +| `Float` | `double` | +| `ID` | `string` | +| `Int` | `int32` | +| `JsonSafeLong` | `int64` | +| `LocalTime` | `string` | +| `LongString` | `int64` | +| `String` | `string` | +| `TimeZone` | `string` | +| `Untyped` | `string` | + +Additionally: +- `DateTime` uses the [well-known `Timestamp` type](https://protobuf.dev/reference/protobuf/google.protobuf/#timestamp); + `schema.proto` imports `google/protobuf/timestamp.proto` automatically. Note that a `Timestamp` + is a UTC instant, so a publisher's original UTC offset is not preserved. +- `string`-typed temporal scalars (`Date`, `LocalTime`, `TimeZone`) are wider than the + ElasticGraph types they carry, so generated fields of these types document the expected format + in a comment (e.g. `// ISO 8601 date, e.g. "2024-11-25"`). Values are validated when events + are ingested, just as with JSON ingestion. +- List types become `repeated` fields. +- Lists of lists (e.g. `[[Float!]!]!`) are not supported because Protocol Buffers cannot represent + them directly. Schema artifact generation raises an error identifying the unsupported field. +- Enum types generate `enum` definitions whose values are prefixed with the enum type name in `UPPER_SNAKE_CASE`, including a zero-valued `*_UNSPECIFIED` entry. diff --git a/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec b/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec index 7ef526536..e691205de 100644 --- a/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec +++ b/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec @@ -36,5 +36,8 @@ Gem::Specification.new do |spec| spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION + # This gem's schema-definition extension code references `elasticgraph-schema_definition`, but + # applications load it through schema-definition tasks after `elasticgraph-schema_definition` is already + # available. Keeping this as a development dependency avoids a runtime dependency cycle. spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb index 761fea495..065d855d7 100644 --- a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb @@ -6,7 +6,12 @@ # # frozen_string_literal: true +require "elastic_graph/errors" require "elastic_graph/proto_ingestion" +require "elastic_graph/proto_ingestion/schema_definition/factory_extension" +require "elastic_graph/proto_ingestion/schema_definition/identifier" +require "elastic_graph/proto_ingestion/schema_definition/schema" +require "elastic_graph/proto_ingestion/schema_definition/state_extension" module ElasticGraph module ProtoIngestion @@ -16,10 +21,66 @@ module ProtoIngestion module SchemaDefinition # Module designed to be extended onto an {ElasticGraph::SchemaDefinition::API} instance # to enable protobuf schema artifact generation. - # - # @note The protobuf schema artifact generation logic has not been implemented yet, so - # extending this module is currently a no-op. module APIExtension + # Wires up the protobuf extensions when this module is extended onto an API instance. + # + # @param api [ElasticGraph::SchemaDefinition::API] the API instance to extend + # @return [void] + # @api private + def self.extended(api) + api.state.extend(StateExtension) + api.factory.extend(FactoryExtension) + end + + # Configures protobuf artifact generation behavior. + # + # @param package_name [String] proto package name to emit + # @param syntax [Symbol] `:proto3` (default) or `:proto2` + # @param headers [Array] file-level lines rendered verbatim after the package declaration + # @return [void] + # + # @example Set the proto package name + # ElasticGraph.define_schema do |schema| + # schema.proto_schema_artifacts package_name: "myapp.events.v1" + # end + def proto_schema_artifacts(package_name:, syntax: :proto3, headers: []) + if !package_name.is_a?(String) || package_name.empty? + raise Errors::SchemaError, "`package_name` must be a non-empty String" + end + unless Schema::SUPPORTED_SYNTAXES.include?(syntax.to_s) + raise Errors::SchemaError, "`syntax` must be one of #{Schema::SUPPORTED_SYNTAXES.inspect}, got: #{syntax.inspect}" + end + if !headers.is_a?(Array) || headers.any? { |header| !header.is_a?(String) } + raise Errors::SchemaError, "`headers` must be an Array of Strings" + end + + ingestion_state = proto_ingestion_state + ingestion_state.package_name = Identifier.validate_package_name(package_name) + ingestion_state.syntax = syntax + ingestion_state.headers = headers + nil + end + + # Configures proto field-number mappings directly from a hash. + # Useful for tests and advanced use cases where mappings are sourced outside artifacts. + # When artifacts are dumped, mappings from the existing `proto_field_numbers.yaml` artifact + # are loaded automatically; this method does not need to be called in that case. + # + # @param proto_field_number_mappings [Hash] + # @return [void] + def configure_proto_field_number_mappings(proto_field_number_mappings) + proto_ingestion_state.field_number_mappings = proto_field_number_mappings + nil + end + + private + + # Returns this gem's state container. Centralizes the Steep cast that's needed because + # Steep can't see the `extend(StateExtension)` applied at runtime in `extended`. + def proto_ingestion_state + extension_state = state # : ElasticGraph::SchemaDefinition::State & StateExtension + extension_state.proto_ingestion_state + end end end end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb new file mode 100644 index 000000000..df7fb9da7 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb @@ -0,0 +1,153 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/results_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Extension module applied to Factory to add proto support. + module FactoryExtension + # Default protobuf options applied to ElasticGraph's built-in scalar types as they are constructed. + BUILT_IN_SCALAR_PROTO_OPTIONS_BY_NAME = { + "Boolean" => {type: "bool"}, + "Cursor" => {type: "string"}, + "Date" => {type: "string", comment: %(ISO 8601 date, e.g. "2024-11-25")}, + "DateTime" => {type: "google.protobuf.Timestamp", import: "google/protobuf/timestamp.proto"}, + "Float" => {type: "double"}, + "ID" => {type: "string"}, + "Int" => {type: "int32"}, + "JsonSafeLong" => {type: "int64"}, + "LocalTime" => {type: "string", comment: %(ISO 8601 local time, e.g. "14:23:12")}, + "LongString" => {type: "int64"}, + "String" => {type: "string"}, + "TimeZone" => {type: "string", comment: %(IANA time zone identifier, e.g. "America/Los_Angeles")}, + "Untyped" => {type: "string"} + }.freeze + + # Creates a new enum type with proto extensions. + # + # @param name [String] enum type name + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::EnumType] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::EnumType] + def new_enum_type(name) + super(name) do |type| + extended_type = type.extend(SchemaElements::EnumTypeExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumType & SchemaElements::EnumTypeExtension + yield extended_type if block_given? + register_proto_type_name(extended_type) + end + end + + # Creates a new enum value with proto extensions. + # + # @param name [String] enum value name + # @param original_name [String] enum value name before overrides + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::EnumValue] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::EnumValue] + def new_enum_value(name, original_name) + super(name, original_name) do |value| + extended_value = value.extend(SchemaElements::EnumValueExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & SchemaElements::EnumValueExtension + yield extended_value if block_given? + end + end + + # Creates a new interface type with proto extensions. + # + # @param name [String] interface type name + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType] + def new_interface_type(name) + super(name) do |type| + extended_type = type.extend(SchemaElements::ObjectInterfaceAndUnionExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType & SchemaElements::ObjectInterfaceAndUnionExtension + yield extended_type if block_given? + register_proto_type_name(extended_type) + end + end + + # Creates a new object type with proto extensions. + # + # @param name [String] object type name + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::ObjectType] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::ObjectType] + def new_object_type(name) + super(name) do |type| + extended_type = type.extend(SchemaElements::ObjectInterfaceAndUnionExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::ObjectType & SchemaElements::ObjectInterfaceAndUnionExtension + yield extended_type if block_given? + register_proto_type_name(extended_type) + end + end + + # Creates a new scalar type with proto extensions. + # + # @param name [String] scalar type name + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::ScalarType] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::ScalarType] + def new_scalar_type(name) + super(name) do |type| + extended_type = type.extend(SchemaElements::ScalarTypeExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension + + if state.initially_registered_built_in_types.empty? && + (proto_options = BUILT_IN_SCALAR_PROTO_OPTIONS_BY_NAME[name.to_s]) + extended_type.protobuf( + type: proto_options.fetch(:type), + import: proto_options[:import], + comment: proto_options[:comment] + ) + end + + yield extended_type if block_given? + extended_type.finalize_protobuf_configuration! + end + end + + # Creates a new union type with proto extensions. + # + # @param name [String] union type name + # @yield [ElasticGraph::SchemaDefinition::SchemaElements::UnionType] + # @return [ElasticGraph::SchemaDefinition::SchemaElements::UnionType] + def new_union_type(name) + super(name) do |type| + extended_type = type.extend(SchemaElements::ObjectInterfaceAndUnionExtension) # : ::ElasticGraph::SchemaDefinition::SchemaElements::UnionType & SchemaElements::ObjectInterfaceAndUnionExtension + yield extended_type if block_given? + register_proto_type_name(extended_type) + end + end + + # Creates a new results object and extends it with proto generation APIs. + # + # @return [ElasticGraph::SchemaDefinition::Results] + def new_results + super.tap do |results| + results.extend ResultsExtension + end + end + + # Creates a new schema artifact manager and extends it with proto artifact support. + # + # @return [ElasticGraph::SchemaDefinition::SchemaArtifactManager] + def new_schema_artifact_manager(...) + super.tap do |manager| + manager.extend SchemaArtifactManagerExtension + end + end + + private + + def register_proto_type_name(type) + extension_state = state # : ElasticGraph::SchemaDefinition::State & StateExtension + extension_state.proto_ingestion_state.register_proto_type_name(type.proto_name, type.name) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/identifier.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/identifier.rb new file mode 100644 index 000000000..cb2647448 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/identifier.rb @@ -0,0 +1,86 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Helpers for rendering Protocol Buffers identifiers while avoiding keyword conflicts. + class Identifier + # Matches a single valid protobuf identifier (e.g. one segment of a package name). + # https://protobuf.com/docs/language-spec#identifiers-and-keywords + VALID_IDENTIFIER = /\A[A-Za-z_][A-Za-z0-9_]*\z/ + + # @dynamic self.enum_name, self.field_name, self.enum_value_name + + class << self + # Validates a protobuf package identifier. + # + # @param name [#to_s] + # @return [String] + def validate_package_name(name) + name = name.to_s + segments = name.split(".", -1) + + if segments.empty? || segments.any? { |segment| !VALID_IDENTIFIER.match?(segment) } + raise Errors::SchemaError, "`package_name` must be a dot-separated list of protobuf identifiers " \ + "(each starting with a letter or underscore, containing only letters, digits, and underscores), " \ + "got: #{name.inspect}." + end + + name + end + + # Builds a protobuf message identifier. + # + # @param name [#to_s] + # @return [String] + def message_name(name) + escape_keyword(name.to_s) + end + + # Builds a protobuf enum identifier. + # + # @return [String] + alias_method :enum_name, :message_name + + # Builds a protobuf field identifier. + # + # @return [String] + alias_method :field_name, :message_name + + # Builds a protobuf enum value identifier. + # + # @return [String] + alias_method :enum_value_name, :message_name + + # Escapes protobuf reserved keywords by suffixing them with an underscore. + # + # @param identifier [String] + # @return [String] + private + + def escape_keyword(identifier) + return identifier unless PROTO_KEYWORDS.include?(identifier) + "#{identifier}_" + end + end + + # Reserved words in protobuf syntax that cannot be used as identifiers verbatim. + # + # @return [Set] + PROTO_KEYWORDS = ::Set[ + "bool", "bytes", "double", "enum", "false", "fixed32", "fixed64", "float", "import", "int32", "int64", "map", + "message", "oneof", "option", "package", "public", "repeated", "reserved", "rpc", "service", "sfixed32", "sfixed64", + "sint32", "sint64", "stream", "string", "syntax", "to", "true", "uint32", "uint64", "weak" + ].freeze + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb new file mode 100644 index 000000000..248eae6ad --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rb @@ -0,0 +1,36 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Holds the proto ingestion extension's schema definition state. + # + # @private + class ProtoIngestionState < ::Struct.new(:package_name, :field_number_mappings, :syntax, :headers) + def initialize(...) + super + @type_name_by_proto_name = {} + end + + # Registers the protobuf name for a schema type, raising immediately if it collides with a prior type. + # + # @return [void] + def register_proto_type_name(proto_name, type_name) + existing_type_name = @type_name_by_proto_name.fetch(proto_name, type_name) + @type_name_by_proto_name[proto_name] = existing_type_name + return if existing_type_name == type_name + + raise Errors::SchemaError, "Type names `#{existing_type_name}` and `#{type_name}` both map to the same proto type name `#{proto_name}`." + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb new file mode 100644 index 000000000..82909e64c --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/results_extension.rb @@ -0,0 +1,54 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/schema" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Extension module for {ElasticGraph::SchemaDefinition::Results} that adds proto schema generation support. + module ResultsExtension + # Returns the generated proto schema. + # + # @return [String] complete `proto3` schema file contents + def proto_schema + @proto_schema ||= protobuf_schema_generator.to_proto + end + + # Returns proto field-number mappings suitable for artifact storage. + # + # @return [Hash] + def proto_field_number_mappings + # Ensure generation has occurred before reading mappings from the generator. + proto_schema + protobuf_schema_generator.field_number_mappings_for_artifact + end + + private + + def protobuf_schema_generator + @protobuf_schema_generator ||= begin + # The cast is needed because Steep can't see the `extend(StateExtension)` applied at + # runtime in {APIExtension.extended}. + extension_state = state # : ElasticGraph::SchemaDefinition::State & StateExtension + ingestion_state = extension_state.proto_ingestion_state + + Schema.new( + state: extension_state, + all_types: all_types, + package_name: ingestion_state.package_name, + proto_field_number_mappings: ingestion_state.field_number_mappings, + syntax: ingestion_state.syntax, + headers: ingestion_state.headers + ) + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb new file mode 100644 index 000000000..e89e5e685 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb @@ -0,0 +1,445 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/proto_ingestion/schema_definition/identifier" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Builds a `proto2` or `proto3` schema string from an ElasticGraph schema definition. + class Schema + # Internal representation of a stored field-number mapping. + # + # @!attribute [r] field_number + # @return [Integer] + # @!attribute [r] name_in_index + # @return [String] + FieldNumberMapping = ::Data.define(:field_number, :name_in_index) + + # The largest field number protobuf allows (2^29 - 1). + MAX_FIELD_NUMBER = 536_870_911 + # Field numbers protobuf reserves for its own implementation; they may not be used as field tags. + RESERVED_FIELD_NUMBER_RANGE = 19_000..19_999 + # The largest enum value number protobuf allows (the int32 maximum). + MAX_ENUM_VALUE_NUMBER = 2_147_483_647 + # Protobuf syntaxes this generator can emit. + SUPPORTED_SYNTAXES = %w[proto2 proto3].freeze + + # @param state [ElasticGraph::SchemaDefinition::State] + # @param all_types [Array] + # @param package_name [String] + # @param proto_field_number_mappings [Hash] + def initialize( + state:, + all_types:, + package_name:, + proto_field_number_mappings: {}, + syntax: :proto3, + headers: [] + ) + @syntax = syntax.to_s + @headers = headers + @state = state + @all_types = all_types + @package_name = Identifier.validate_package_name(package_name) + @proto_field_number_mappings_by_message = normalize_proto_field_number_mappings(proto_field_number_mappings) + @proto_enum_value_numbers_by_enum = normalize_proto_enum_value_number_mappings(proto_field_number_mappings) + @used_field_numbers_by_message = {} + @used_enum_value_numbers_by_enum = {} + end + + # Renders the schema as a valid `proto3` file. + # + # @return [String] + def to_proto + types = proto_types + return "" if types.empty? + + sections = [ + %(syntax = "#{@syntax}";), + "package #{@package_name};", + *render_headers, + *render_imports(types), + render_definitions(types) + ] + + sections.join("\n\n") + "\n" + end + + # Exposes normalized field-number and enum-value-number mappings for writing to artifact YAML. + # + # @return [Hash] + def field_number_mappings_for_artifact + { + "messages" => @proto_field_number_mappings_by_message + .sort_by { |message_name, _| message_name } + .to_h do |message_name, field_numbers| + [message_name, { + "fields" => field_numbers.sort_by { |field_name, mapping| [mapping.field_number, field_name] }.to_h do |field_name, mapping| + artifact_mapping = + if mapping.name_in_index == field_name + mapping.field_number + else + { + "field_number" => mapping.field_number, + "name_in_index" => mapping.name_in_index + } + end + + [field_name, artifact_mapping] + end + }] + end, + "enums" => @proto_enum_value_numbers_by_enum + .sort_by { |enum_name, _| enum_name } + .to_h do |enum_name, value_numbers| + [enum_name, { + "values" => value_numbers.sort_by { |value_name, number| [number, value_name] }.to_h + }] + end + } + end + + # Returns the stable protobuf number for a message field. + # + # @api private + def field_number_for(message_name:, type_name:, public_field_name:, name_in_index:) + mappings_for_message = @proto_field_number_mappings_by_message[message_name] ||= {} + used_numbers = @used_field_numbers_by_message[message_name] ||= ::Set.new(mappings_for_message.values.map(&:field_number)) + + mapping = mappings_for_message.fetch(public_field_name) do + migrate_renamed_field_mapping(mappings_for_message, type_name: type_name, public_field_name: public_field_name) || begin + next_field_number = next_available_field_number(used_numbers) + used_numbers << next_field_number + FieldNumberMapping.new(field_number: next_field_number, name_in_index: name_in_index) + end + end + + mapping = FieldNumberMapping.new(field_number: mapping.field_number, name_in_index: name_in_index) if mapping.name_in_index != name_in_index + mappings_for_message[public_field_name] = mapping + + duplicate_field_name = mappings_for_message.find do |mapped_field_name, mapped_field_number| + mapped_field_name != public_field_name && mapped_field_number.field_number == mapping.field_number + end&.first + + if duplicate_field_name + raise Errors::SchemaError, "Protobuf field-number mapping collision in message `#{message_name}`: " \ + "`#{duplicate_field_name}` and `#{public_field_name}` are both mapped to field number #{mapping.field_number}." + end + + mapping.field_number + end + + # Returns the stable protobuf numbers for an enum's values. + # + # @api private + def enum_value_numbers_for(enum_name, value_names) + value_numbers = @proto_enum_value_numbers_by_enum[enum_name] ||= {} + used_numbers = used_enum_value_numbers_for(enum_name, value_numbers) + + value_names.to_h do |value_name| + number = value_numbers.fetch(value_name) do + next_available_enum_value_number(used_numbers).tap do |allocated| + used_numbers << allocated + value_numbers[value_name] = allocated + end + end + [value_name, number] + end + end + + # Returns the label for a protobuf field under the configured syntax. + # + # @api private + def field_label(repeated) + return repeated ? "repeated " : "optional " if @syntax == "proto2" + repeated ? "repeated " : nil + end + + private + + # Selects the indexed root types and every type transitively referenced by their protobuf + # representations. All traversal state is local so repeated calls are independent. + def proto_types + types_to_visit = _ = @state.indexed_types_by_index_name.values.dup + type_names_to_render = ::Set.new + + while (type = types_to_visit.shift) + next unless type_names_to_render.add?(type.name) + + types_to_visit.concat(type.referenced_proto_types) + end + + @all_types.select do |type| + type_names_to_render.include?(type.name) && type.respond_to?(:to_proto) + end + end + + def render_definitions(types) + types + .sort_by { |type| [(type.proto_definition_kind == :enum) ? 0 : 1, type.proto_name] } + .filter_map do |type| + type.proto_definition_kind ? type.to_proto(self) : type.to_proto + end + .join("\n\n") + end + + def render_imports(types) + imports = types.filter_map do |type| + type.protobuf_import if type.respond_to?(:protobuf_import) + end.uniq.sort + + imports.empty? ? [] : [imports.map { |import| %(import "#{import}";) }.join("\n")] + end + + def render_headers + @headers.empty? ? [] : [@headers.join("\n")] + end + + def valid_field_number?(number) + number.between?(1, MAX_FIELD_NUMBER) && !RESERVED_FIELD_NUMBER_RANGE.cover?(number) + end + + # Returns the smallest valid field tag not yet present in `used_numbers`, skipping the + # protobuf-reserved 19000..19999 range. Callers maintain the set (numbers loaded from a + # previously dumped artifact plus numbers allocated so far) so that allocation is + # constant-time per candidate instead of rescanning all mappings. + def next_available_field_number(used_numbers) + candidate = 1 + candidate += 1 while used_numbers.include?(candidate) || RESERVED_FIELD_NUMBER_RANGE.cover?(candidate) + candidate + end + + # Returns the smallest positive enum value number not yet present in `used_numbers`. + # Unlike field tags, enum value numbers have no protobuf-reserved range. + def next_available_enum_value_number(used_numbers) + candidate = 1 + candidate += 1 while used_numbers.include?(candidate) + candidate + end + + def used_enum_value_numbers_for(enum_name, value_numbers) + @used_enum_value_numbers_by_enum[enum_name] ||= begin + duplicated_numbers = value_numbers.group_by { |_, number| number }.select { |_, entries| entries.size > 1 } + + duplicated_numbers.each_value do |entries| + value_names = entries.map(&:first).sort.map { |name| "`#{name}`" }.join(" and ") + raise Errors::SchemaError, "Protobuf enum value-number mapping collision in enum `#{enum_name}`: " \ + "#{value_names} are both mapped to number #{entries.first.last}." + end + + ::Set.new(value_numbers.values) + end + end + + def migrate_renamed_field_mapping(mappings_for_message, type_name:, public_field_name:) + renames_for_type = renamed_public_field_names_by_type_name.fetch(type_name) { return nil } + old_field_names = renames_for_type.fetch(public_field_name) { return nil } + + old_field_names.each do |old_field_name| + return mappings_for_message.delete(old_field_name) if mappings_for_message.key?(old_field_name) + end + + nil + end + + def normalize_proto_field_number_mappings(raw_mappings) + return {} if raw_mappings.nil? + unless raw_mappings.is_a?(Hash) + raise Errors::SchemaError, "Protobuf field-number mappings must be a Hash, got: #{raw_mappings.class}." + end + + messages_hash = + if raw_mappings.key?("messages") + raw_mappings.fetch("messages") + elsif raw_mappings.key?(:messages) + raw_mappings.fetch(:messages) + elsif raw_mappings.key?("enums") || raw_mappings.key?(:enums) + # The hash uses the sectioned artifact format but only maps enum value numbers. + {} # : ::Hash[untyped, untyped] + else + raw_mappings + end + + unless messages_hash.is_a?(Hash) + raise Errors::SchemaError, "Protobuf field-number mappings must have a `messages` Hash." + end + + normalized = {} # : ::Hash[::String, fieldNumberMappingsByFieldName] + + messages_hash.each do |message_name, field_numbers| + unless field_numbers.is_a?(Hash) + raise Errors::SchemaError, "Field-number mapping for message `#{message_name}` must be a Hash." + end + + normalized_fields = + if field_numbers.key?("fields") + field_numbers.fetch("fields") + elsif field_numbers.key?(:fields) + field_numbers.fetch(:fields) + else + field_numbers + end + + unless normalized_fields.is_a?(Hash) + raise Errors::SchemaError, "Field-number mapping for message `#{message_name}` must contain a `fields` Hash." + end + + normalized_message_name = message_name.to_s + normalized_field_numbers = {} # : fieldNumberMappingsByFieldName + + normalized_fields.each do |field_name, field_number_or_mapping| + normalized_field_name = field_name.to_s + normalized_field_number, normalized_name_in_index = normalize_field_number_mapping_entry( + normalized_message_name, + normalized_field_name, + field_number_or_mapping + ) + + unless valid_field_number?(normalized_field_number) + raise Errors::SchemaError, "Field-number mapping for `#{normalized_message_name}.#{normalized_field_name}` " \ + "must be a valid protobuf field number (1 to #{MAX_FIELD_NUMBER}, excluding the reserved " \ + "#{RESERVED_FIELD_NUMBER_RANGE.begin}-#{RESERVED_FIELD_NUMBER_RANGE.end} range), got: #{field_number_or_mapping.inspect}." + end + + normalized_field_numbers[normalized_field_name] = FieldNumberMapping.new( + field_number: normalized_field_number, + name_in_index: normalized_name_in_index + ) + end + + normalized[normalized_message_name] = normalized_field_numbers + end + + normalized + end + + def normalize_proto_enum_value_number_mappings(raw_mappings) + return {} if raw_mappings.nil? + + enums_hash = + if raw_mappings.is_a?(Hash) && raw_mappings.key?("enums") + raw_mappings.fetch("enums") + elsif raw_mappings.is_a?(Hash) && raw_mappings.key?(:enums) + raw_mappings.fetch(:enums) + else + return {} + end + + unless enums_hash.is_a?(Hash) + raise Errors::SchemaError, "Protobuf enum value-number mappings must have an `enums` Hash." + end + + normalized = {} # : ::Hash[::String, ::Hash[::String, ::Integer]] + + enums_hash.each do |enum_name, value_numbers| + values_hash = + if value_numbers.is_a?(Hash) && value_numbers.key?("values") + value_numbers.fetch("values") + elsif value_numbers.is_a?(Hash) && value_numbers.key?(:values) + value_numbers.fetch(:values) + else + value_numbers + end + + unless values_hash.is_a?(Hash) + raise Errors::SchemaError, "Enum value-number mapping for enum `#{enum_name}` must contain a `values` Hash." + end + + normalized_enum_name = enum_name.to_s + normalized_value_numbers = {} # : ::Hash[::String, ::Integer] + + values_hash.each do |value_name, value_number| + normalized_number = integral_number(value_number) + + if normalized_number.nil? || normalized_number <= 0 || normalized_number > MAX_ENUM_VALUE_NUMBER + raise Errors::SchemaError, "Enum value-number mapping for `#{normalized_enum_name}.#{value_name}` " \ + "must be a positive integer no greater than #{MAX_ENUM_VALUE_NUMBER} " \ + "(0 is reserved for the `_UNSPECIFIED` value), got: #{value_number.inspect}." + end + + normalized_value_numbers[value_name.to_s] = normalized_number + end + + normalized[normalized_enum_name] = normalized_value_numbers + end + + normalized + end + + def normalize_field_number_mapping_entry(message_name, field_name, field_number_or_mapping) + if field_number_or_mapping.is_a?(Hash) + raw_field_number = + if field_number_or_mapping.key?("field_number") + field_number_or_mapping.fetch("field_number") + elsif field_number_or_mapping.key?(:field_number) + field_number_or_mapping.fetch(:field_number) + else + raise Errors::SchemaError, "Field-number mapping for `#{message_name}.#{field_name}` must include `field_number`." + end + + raw_name_in_index = + if field_number_or_mapping.key?("name_in_index") + field_number_or_mapping.fetch("name_in_index") + elsif field_number_or_mapping.key?(:name_in_index) + field_number_or_mapping.fetch(:name_in_index) + else + field_name + end + + unless raw_name_in_index.is_a?(String) || raw_name_in_index.is_a?(Symbol) + raise Errors::SchemaError, "Field-number mapping for `#{message_name}.#{field_name}` " \ + "must use a String or Symbol `name_in_index`, got: #{raw_name_in_index.inspect}." + end + + [integral_field_number(message_name, field_name, raw_field_number), raw_name_in_index.to_s] + else + [integral_field_number(message_name, field_name, field_number_or_mapping), field_name] + end + end + + def integral_field_number(message_name, field_name, raw_field_number) + number = integral_number(raw_field_number) + + if number.nil? + raise Errors::SchemaError, "Field-number mapping for `#{message_name}.#{field_name}` " \ + "must be an integer, got: #{raw_field_number.inspect}." + end + + number + end + + # Coerces a raw mapping number to an Integer without truncating: only `Integer` values and + # strings that parse as integers are accepted; anything else (including floats, which + # `Kernel#Integer` would silently truncate) returns `nil`. + def integral_number(raw) + case raw + when ::Integer then raw + when ::String then raw.match?(/\A-?\d+\z/) ? Integer(raw) : nil + end + end + + def renamed_public_field_names_by_type_name + @renamed_public_field_names_by_type_name ||= @state.renamed_fields_by_type_name_and_old_field_name.to_h do |type_name, old_to_new| + current_to_old = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::String]] + + old_to_new.each do |old_field_name, renamed_field| + current_to_old[renamed_field.name] << old_field_name + end + + [type_name, current_to_old] + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rb new file mode 100644 index 000000000..2a0bb322d --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rb @@ -0,0 +1,66 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion" +require "yaml" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Extension module for {ElasticGraph::SchemaDefinition::SchemaArtifactManager} that adds + # proto artifact generation support. + # + # @private + module SchemaArtifactManagerExtension + private + + # Overrides the base `artifacts_from_schema_def` method to add proto artifacts. + def artifacts_from_schema_def + protobuf_load_existing_field_number_mappings + + base_artifacts = super + proto_schema = protobuf_schema_definition_results.proto_schema + return base_artifacts if proto_schema.empty? + + base_artifacts + [ + new_yaml_artifact( + PROTO_FIELD_NUMBERS_FILE, + protobuf_schema_definition_results.proto_field_number_mappings, + extra_comment_lines: [ + "This file reserves protobuf field and enum value numbers to keep them stable over time.", + "Do not renumber existing entries." + ] + ), + new_raw_artifact(PROTO_SCHEMA_FILE, proto_schema.chomp, comment_prefix: "//") + ] + end + + # Returns the wrapped {ElasticGraph::SchemaDefinition::Results} narrowed to include this + # gem's `ResultsExtension`. Centralizes the Steep cast that's needed because Steep can't + # see the `extend(ResultsExtension)` applied at runtime. + def protobuf_schema_definition_results + schema_definition_results # : ElasticGraph::SchemaDefinition::Results & ResultsExtension + end + + def proto_ingestion_state + extension_state = protobuf_schema_definition_results.state # : ElasticGraph::SchemaDefinition::State & StateExtension + extension_state.proto_ingestion_state + end + + # Seeds the schema generator with the field-number mappings from the previously dumped + # artifact (if any) so that field numbers remain stable across dumps. + def protobuf_load_existing_field_number_mappings + full_path = ::File.join(@schema_artifacts_directory, PROTO_FIELD_NUMBERS_FILE) + return unless ::File.exist?(full_path) + + proto_ingestion_state.field_number_mappings = ::YAML.safe_load_file(full_path, aliases: false) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb new file mode 100644 index 000000000..2652b66e5 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rb @@ -0,0 +1,112 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/proto_ingestion/schema_definition/identifier" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation" +require "elastic_graph/support/casing" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Protobuf schema definition extensions for ElasticGraph schema elements. + module SchemaElements + # Extends EnumType with proto field type conversion. + module EnumTypeExtension + # Defines an enum value and immediately validates its protobuf name. + # + # @return [void] + def value(value_name, &block) + super + new_value = values_by_name.values.last # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension + new_proto_name = new_value.proto_name(enum_value_prefix) + zero_value_name = "#{enum_value_prefix}_UNSPECIFIED" + + if new_proto_name == zero_value_name + raise Errors::SchemaError, "Enum `#{name}` value `#{new_value.name}` maps to proto enum value name " \ + "`#{new_proto_name}`, which conflicts with the generated zero value `#{zero_value_name}`." + end + + duplicate = values_by_name.values.find do |raw_value| + existing_value = raw_value # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension + !existing_value.equal?(new_value) && existing_value.proto_name(enum_value_prefix) == new_proto_name + end + + if duplicate + raise Errors::SchemaError, "Enum `#{name}` values `#{duplicate.name}` and `#{new_value.name}` " \ + "map to duplicate proto enum value name `#{new_proto_name}`." + end + + nil + end + + # Renders this enum's protobuf definition. + # + # @return [String] + def to_proto(schema) + render_proto_enum(schema) + end + + # Returns the kind used to order this definition in a protobuf schema. + # + # @return [Symbol] + def proto_definition_kind + :enum + end + + # Returns the schema types referenced by this definition. + # + # @return [Array] + def referenced_proto_types + [] + end + + # Returns this enum type's name in protobuf schemas. + # + # @return [String] + def proto_name + @proto_name ||= Identifier.enum_name(name) + end + + # @private + def configure_derived_scalar_type(scalar_type) + super + proto_scalar_type = scalar_type # : ::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & ScalarTypeExtension + proto_scalar_type.protobuf type: proto_name + end + + private + + def render_proto_enum(schema) + values = values_by_name.values + value_numbers = schema.enum_value_numbers_for(proto_name, values_by_name.keys) + + zero_value_name = "#{enum_value_prefix}_UNSPECIFIED" + + lines = ProtoDocumentation.comment_lines_for(doc_comment) + lines << "enum #{proto_name} {" + lines << " // The default value when no enum value has been explicitly set. Do not use this value." + lines << " // See https://protobuf.dev/programming-guides/proto3/#enum-default." + lines << " #{zero_value_name} = 0;" + values.each do |raw_value| + value = raw_value # : ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension + lines.concat(ProtoDocumentation.comment_lines_for(value.doc_comment, indent: " ")) + lines << " #{value.proto_name(enum_value_prefix)} = #{value_numbers.fetch(value.name)};" + end + lines << "}" + lines.join("\n") + end + + def enum_value_prefix + Support::Casing.to_upper_snake(name) + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rb new file mode 100644 index 000000000..3c597d914 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rb @@ -0,0 +1,29 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/identifier" +require "elastic_graph/support/casing" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + # Extends enum values with protobuf naming behavior. + module EnumValueExtension + # Returns this enum value's name in protobuf schemas. + # + # @param enum_value_prefix [String] normalized prefix of the containing enum + # @return [String] + def proto_name(enum_value_prefix) + @proto_name ||= Identifier.enum_value_name("#{enum_value_prefix}_#{Support::Casing.to_upper_snake(name)}") + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb new file mode 100644 index 000000000..6e983886c --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rb @@ -0,0 +1,151 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" +require "elastic_graph/proto_ingestion/schema_definition/identifier" +require "elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation" +require "elastic_graph/support/casing" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + # Extends object/interface/union types with proto field type conversion. + module ObjectInterfaceAndUnionExtension + # Renders this type's protobuf message definition. + # + # @return [String] + def to_proto(schema) + render_proto_message(schema, proto_name) + end + + # Returns the kind used to order this definition in a protobuf schema. + # + # @return [Symbol] + def proto_definition_kind + :message + end + + # Returns the schema types referenced by this definition. + # + # @return [Array] + def referenced_proto_types + if abstract? + abstract_type = _ = self + abstract_type.recursively_resolve_subtypes + else + proto_fields.map do |_, field| + _ = field.type.fully_unwrapped.resolved + end + end + end + + # Returns this type's name in protobuf schemas. + # + # @return [String] + def proto_name + @proto_name ||= Identifier.message_name(name) + end + + private + + def render_proto_message(schema, message_name) + return render_proto_oneof(schema, message_name) if abstract? + + fields = proto_fields + field_names = fields.map { |_, field| Identifier.field_name(field.name) } + duplicate_names = field_names.tally.select { |_, count| count > 1 } + if duplicate_names.any? + raise Errors::SchemaError, "Type `#{name}` maps to duplicate proto field names: #{duplicate_names.keys.sort.join(", ")}." + end + + lines = ProtoDocumentation.comment_lines_for(doc_comment) + lines << "message #{message_name} {" + fields.each do |schema_field, field| + field_name = Identifier.field_name(field.name) + repeated, field_type, type_comment = proto_field_type_for(field.type, context_field_name: field.name) + field_number = schema.field_number_for( + message_name: message_name, + type_name: name, + public_field_name: schema_field.name, + name_in_index: field.name_in_index + ) + label = schema.field_label(repeated) + line = " #{label}#{field_type} #{field_name} = #{field_number};" + comment_parts = [] # : ::Array[::String] + comment_parts << "source name: #{field.name}" if field_name != field.name + comment_parts << type_comment if type_comment + line += " // #{comment_parts.join("; ")}" if comment_parts.any? + lines.concat(ProtoDocumentation.comment_lines_for(schema_field.doc_comment, indent: " ")) + lines << line + end + lines << "}" + lines.join("\n") + end + + def render_proto_oneof(schema, message_name) + # @type var abstract_type: ::ElasticGraph::SchemaDefinition::Mixins::HasSubtypes + abstract_type = _ = self + lines = ProtoDocumentation.comment_lines_for(doc_comment) + lines << "message #{message_name} {" + lines << " oneof value {" + abstract_type.recursively_resolve_subtypes.each do |subtype| + subtype_name = (_ = subtype).proto_name + field_name = Identifier.field_name(Support::Casing.to_upper_snake(subtype_name).downcase) + field_number = schema.field_number_for( + message_name: message_name, + type_name: name, + public_field_name: field_name, + name_in_index: field_name + ) + lines << " #{subtype_name} #{field_name} = #{field_number};" + end + lines << " }" + lines << "}" + lines.join("\n") + end + + def proto_fields + indexing_fields_by_name_in_index.values.filter_map do |schema_field| + next if schema_field.name == "__typename" + + indexing_field = schema_field.to_indexing_field # : ElasticGraph::SchemaDefinition::Indexing::Field + [schema_field, indexing_field] # : [::ElasticGraph::SchemaDefinition::SchemaElements::Field, ::ElasticGraph::SchemaDefinition::Indexing::Field] + end + end + + def proto_field_type_for(type_ref, context_field_name:) + list_depth, base_type_ref = list_depth_and_base_type(type_ref) + + if list_depth > 1 + raise Errors::SchemaError, "Field `#{name}.#{context_field_name}` has type `#{type_ref.name}`, " \ + "but Protocol Buffers cannot represent lists of lists directly. " \ + "`elasticgraph-proto_ingestion` supports fields with at most one list level." + end + + proto_type = _ = base_type_ref.resolved + type_comment = (ScalarTypeExtension === proto_type) ? proto_type.protobuf_comment : nil + [list_depth == 1, proto_type.proto_name, type_comment] + end + + def list_depth_and_base_type(type_ref) + list_depth = 0 + current = type_ref.unwrap_non_null + + while current.list? + list_depth += 1 + current = current.unwrap_list.unwrap_non_null + end + + [list_depth, current] + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rb new file mode 100644 index 000000000..0f4b8f32b --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rb @@ -0,0 +1,33 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + # Formats ElasticGraph schema element documentation as protobuf comments. + # + # @api private + module ProtoDocumentation + # Formats documentation as line comments. + # + # @param doc_comment [String, nil] schema element documentation + # @param indent [String] indentation to put before each comment + # @return [Array] formatted protobuf comment lines + def self.comment_lines_for(doc_comment, indent: "") + return [] unless doc_comment + + doc_comment.chomp.lines(chomp: true).map do |line| + line.empty? ? "#{indent}//" : "#{indent}// #{line}" + end + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb new file mode 100644 index 000000000..5454f586a --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rb @@ -0,0 +1,86 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/errors" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + # Extends ScalarType with proto field type conversion. + module ScalarTypeExtension + # Configured protobuf type (e.g. string, int64, bool). + # @dynamic protobuf_type + attr_reader :protobuf_type + + # Proto file to import for the configured protobuf type, if it is externally defined. + # @dynamic protobuf_import + attr_reader :protobuf_import + + # Comment rendered on generated proto fields of this scalar type. + # @dynamic protobuf_comment + attr_reader :protobuf_comment + + # Configures the protobuf type for this scalar type. + # + # @param type [String] protobuf type name + # @param import [String, nil] proto file to import for an externally defined type + # @param comment [String, nil] comment rendered on generated fields of this type + # @return [void] + def protobuf(type:, import: nil, comment: nil) + @protobuf_type = type + @protobuf_import = import + @protobuf_comment = comment + @proto_name = nil + end + + # Validates that a protobuf type has been configured on this scalar type. GraphQL-only + # scalar types are skipped because they are not part of ingestion. + # + # @return [void] + # @raise [Errors::SchemaError] when missing + def finalize_protobuf_configuration! + return if graphql_only? + proto_name + nil + end + + # Scalars map to protobuf field types and do not render standalone definitions. + # + # @return [nil] + def to_proto + nil + end + + # Returns the kind used to order this definition in a protobuf schema. + # + # @return [nil] + def proto_definition_kind + nil + end + + # Returns the schema types referenced by this definition. + # + # @return [Array] + def referenced_proto_types + [] + end + + # Returns this scalar's name in protobuf schemas. + # + # @return [String] + def proto_name + @proto_name ||= protobuf_type || + raise(Errors::SchemaError, "Protobuf type not configured for scalar type `#{name}`. " \ + 'To proceed, call `protobuf type: "TYPE"` on the scalar type definition.') + end + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb new file mode 100644 index 000000000..c9ccef295 --- /dev/null +++ b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/state_extension.rb @@ -0,0 +1,35 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + # Extension module applied to `ElasticGraph::SchemaDefinition::State` to hold protobuf configuration. + # + # @private + module StateExtension + # @dynamic proto_ingestion_state + attr_reader :proto_ingestion_state + + def self.extended(state) + state.instance_variable_set( + :@proto_ingestion_state, + ProtoIngestionState.new( + package_name: "elasticgraph", + field_number_mappings: {}, + syntax: :proto3, + headers: [] + ) + ) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs index 81e79cec4..5e340ecd5 100644 --- a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs @@ -1,7 +1,18 @@ module ElasticGraph module ProtoIngestion module SchemaDefinition - module APIExtension + module APIExtension: ::ElasticGraph::SchemaDefinition::API + def self.extended: (::ElasticGraph::SchemaDefinition::API & APIExtension) -> void + def proto_schema_artifacts: ( + package_name: ::String, + ?syntax: (::Symbol | ::String), + ?headers: ::Array[::String] + ) -> void + def configure_proto_field_number_mappings: (untyped) -> void + + private + + def proto_ingestion_state: () -> ProtoIngestionState end end end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/factory_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/factory_extension.rbs new file mode 100644 index 000000000..39a29dedc --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/factory_extension.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module FactoryExtension: ::ElasticGraph::SchemaDefinition::Factory + BUILT_IN_SCALAR_PROTO_OPTIONS_BY_NAME: ::Hash[::String, ::Hash[::Symbol, ::String]] + + def new_enum_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumType & SchemaElements::EnumTypeExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::EnumType & SchemaElements::EnumTypeExtension) + def new_enum_value: (::String, ::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & SchemaElements::EnumValueExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & SchemaElements::EnumValueExtension) + def new_interface_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType & SchemaElements::ObjectInterfaceAndUnionExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType & SchemaElements::ObjectInterfaceAndUnionExtension) + def new_object_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::ObjectType & SchemaElements::ObjectInterfaceAndUnionExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::ObjectType & SchemaElements::ObjectInterfaceAndUnionExtension) + def new_scalar_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension) + def new_union_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::UnionType & SchemaElements::ObjectInterfaceAndUnionExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::UnionType & SchemaElements::ObjectInterfaceAndUnionExtension) + def new_results: () -> ::ElasticGraph::SchemaDefinition::Results + def new_schema_artifact_manager: (**untyped) -> ::ElasticGraph::SchemaDefinition::SchemaArtifactManager + + private + + def register_proto_type_name: (untyped) -> void + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/identifier.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/identifier.rbs new file mode 100644 index 000000000..25c69322d --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/identifier.rbs @@ -0,0 +1,20 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + class Identifier + PROTO_KEYWORDS: ::Set[::String] + VALID_IDENTIFIER: ::Regexp + + def self.validate_package_name: (::String) -> ::String + def self.message_name: (::String) -> ::String + def self.enum_name: (::String) -> ::String + def self.field_name: (::String) -> ::String + def self.enum_value_name: (::String) -> ::String + + private + + def self.escape_keyword: (::String) -> ::String + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs new file mode 100644 index 000000000..bf33fd64a --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/proto_ingestion_state.rbs @@ -0,0 +1,25 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + class ProtoIngestionStateSupertype + attr_accessor package_name: ::String + attr_accessor field_number_mappings: untyped + attr_accessor syntax: (::Symbol | ::String) + attr_accessor headers: ::Array[::String] + + def initialize: ( + package_name: ::String, + field_number_mappings: untyped, + syntax: (::Symbol | ::String), + headers: ::Array[::String] + ) -> void + end + + class ProtoIngestionState < ProtoIngestionStateSupertype + def register_proto_type_name: (::String proto_name, ::String type_name) -> void + + @type_name_by_proto_name: ::Hash[::String, ::String] + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/results_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/results_extension.rbs new file mode 100644 index 000000000..2d5339a10 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/results_extension.rbs @@ -0,0 +1,17 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module ResultsExtension : ::ElasticGraph::SchemaDefinition::Results + def proto_schema: () -> ::String + def proto_field_number_mappings: () -> untyped + + private + + def protobuf_schema_generator: () -> Schema + + @proto_schema: ::String? + @protobuf_schema_generator: Schema? + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs new file mode 100644 index 000000000..f786d1464 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema.rbs @@ -0,0 +1,74 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + class Schema + class FieldNumberMapping + attr_reader field_number: ::Integer + attr_reader name_in_index: ::String + + def self.new: (field_number: ::Integer, name_in_index: ::String) -> instance + end + + type fieldNumberMappingsByFieldName = ::Hash[::String, FieldNumberMapping] + + MAX_FIELD_NUMBER: ::Integer + RESERVED_FIELD_NUMBER_RANGE: ::Range[::Integer] + MAX_ENUM_VALUE_NUMBER: ::Integer + SUPPORTED_SYNTAXES: ::Array[::String] + + @syntax: ::String + @headers: ::Array[::String] + @state: ::ElasticGraph::SchemaDefinition::State + @all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] + @package_name: ::String + @proto_field_number_mappings_by_message: ::Hash[::String, fieldNumberMappingsByFieldName] + @proto_enum_value_numbers_by_enum: ::Hash[::String, ::Hash[::String, ::Integer]] + @used_field_numbers_by_message: ::Hash[::String, ::Set[::Integer]] + @used_enum_value_numbers_by_enum: ::Hash[::String, ::Set[::Integer]] + @renamed_public_field_names_by_type_name: ::Hash[::String, ::Hash[::String, ::Array[::String]]]? + + def initialize: ( + state: ::ElasticGraph::SchemaDefinition::State, + all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType], + package_name: ::String, + ?proto_field_number_mappings: untyped, + ?syntax: (::Symbol | ::String), + ?headers: ::Array[::String] + ) -> void + + def to_proto: () -> ::String + def field_number_mappings_for_artifact: () -> ::Hash[::String, untyped] + def field_number_for: ( + message_name: ::String, + type_name: ::String, + public_field_name: ::String, + name_in_index: ::String + ) -> ::Integer + def enum_value_numbers_for: (::String, ::Array[::String]) -> ::Hash[::String, ::Integer] + def field_label: (bool) -> ::String? + + private + + def proto_types: () -> ::Array[untyped] + def render_definitions: (::Array[untyped] types) -> ::String + def render_imports: (::Array[untyped] types) -> ::Array[::String] + def render_headers: () -> ::Array[::String] + def valid_field_number?: (::Integer) -> bool + def next_available_field_number: (::Set[::Integer]) -> ::Integer + def next_available_enum_value_number: (::Set[::Integer]) -> ::Integer + def used_enum_value_numbers_for: (::String, ::Hash[::String, ::Integer]) -> ::Set[::Integer] + def migrate_renamed_field_mapping: ( + fieldNumberMappingsByFieldName, + type_name: ::String, + public_field_name: ::String + ) -> FieldNumberMapping? + def normalize_proto_field_number_mappings: (untyped) -> ::Hash[::String, fieldNumberMappingsByFieldName] + def normalize_proto_enum_value_number_mappings: (untyped) -> ::Hash[::String, ::Hash[::String, ::Integer]] + def normalize_field_number_mapping_entry: (::String, ::String, untyped) -> [::Integer, ::String] + def integral_field_number: (::String, ::String, untyped) -> ::Integer + def integral_number: (untyped) -> ::Integer? + def renamed_public_field_names_by_type_name: () -> ::Hash[::String, ::Hash[::String, ::Array[::String]]] + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rbs new file mode 100644 index 000000000..91342e799 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension.rbs @@ -0,0 +1,14 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaArtifactManagerExtension : ::ElasticGraph::SchemaDefinition::SchemaArtifactManager + private + + def artifacts_from_schema_def: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]] + def protobuf_schema_definition_results: () -> (::ElasticGraph::SchemaDefinition::Results & ResultsExtension) + def proto_ingestion_state: () -> ProtoIngestionState + def protobuf_load_existing_field_number_mappings: () -> void + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs new file mode 100644 index 000000000..5e47f4c57 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension.rbs @@ -0,0 +1,23 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + module EnumTypeExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::EnumType + @proto_name: ::String? + + def value: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue & EnumValueExtension) -> void } -> void + def proto_name: () -> ::String + def to_proto: (Schema) -> ::String + def proto_definition_kind: () -> :enum + def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] + def configure_derived_scalar_type: (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType) -> void + + private + + def render_proto_enum: (Schema) -> ::String + def enum_value_prefix: () -> ::String + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rbs new file mode 100644 index 000000000..c1d2fa996 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_value_extension.rbs @@ -0,0 +1,13 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + module EnumValueExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::EnumValue + @proto_name: ::String? + + def proto_name: (::String enum_value_prefix) -> ::String + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs new file mode 100644 index 000000000..2ef470ad7 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension.rbs @@ -0,0 +1,32 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + module ObjectInterfaceAndUnionExtension : ::ElasticGraph::SchemaDefinition::SchemaElements::ObjectType + @proto_name: ::String? + + def proto_name: () -> ::String + def to_proto: (Schema) -> ::String + def proto_definition_kind: () -> :message + def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] + + private + + def render_proto_message: (Schema, ::String message_name) -> ::String + def render_proto_oneof: (Schema, ::String message_name) -> ::String + def proto_fields: () -> ::Array[[ + ::ElasticGraph::SchemaDefinition::SchemaElements::Field, + ::ElasticGraph::SchemaDefinition::Indexing::Field + ]] + def proto_field_type_for: ( + ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference, + context_field_name: ::String + ) -> [bool, ::String, ::String?] + def list_depth_and_base_type: ( + ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference + ) -> [::Integer, ::ElasticGraph::SchemaDefinition::SchemaElements::TypeReference] + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rbs new file mode 100644 index 000000000..c2cf49bd8 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/proto_documentation.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + module ProtoDocumentation + def self.comment_lines_for: (::String?, ?indent: ::String) -> ::Array[::String] + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs new file mode 100644 index 000000000..89ac66542 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs @@ -0,0 +1,22 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module SchemaElements + module ScalarTypeExtension: ::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType + @proto_name: ::String? + + attr_reader protobuf_type: ::String? + attr_reader protobuf_import: ::String? + attr_reader protobuf_comment: ::String? + + def protobuf: (type: ::String, ?import: ::String?, ?comment: ::String?) -> void + def finalize_protobuf_configuration!: () -> void + def proto_name: () -> ::String + def to_proto: () -> nil + def proto_definition_kind: () -> nil + def referenced_proto_types: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/state_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/state_extension.rbs new file mode 100644 index 000000000..3b29bc557 --- /dev/null +++ b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/state_extension.rbs @@ -0,0 +1,11 @@ +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + module StateExtension: ::ElasticGraph::SchemaDefinition::State + attr_reader proto_ingestion_state: ProtoIngestionState + + def self.extended: (::ElasticGraph::SchemaDefinition::State & StateExtension) -> void + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/integration/elastic_graph/proto_ingestion/schema_definition/rake_tasks_spec.rb b/elasticgraph-proto_ingestion/spec/integration/elastic_graph/proto_ingestion/schema_definition/rake_tasks_spec.rb new file mode 100644 index 000000000..4f702ed4b --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/integration/elastic_graph/proto_ingestion/schema_definition/rake_tasks_spec.rb @@ -0,0 +1,130 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion" +require "elastic_graph/proto_ingestion/schema_definition/api_extension" +require "elastic_graph/schema_definition/rake_tasks" +require "yaml" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe "Protobuf RakeTasks", :rake_task, :in_temp_dir do + describe "schema_artifacts:dump" do + it "dumps proto artifact when indexed types are defined" do + write_proto_schema(table_defs: <<~EOS) + s.object_type "Product" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "products" + end + EOS + + expect { + output = run_rake_with_proto("schema_artifacts:dump") + expect(output.lines).to include(a_string_including("Dumped", PROTO_SCHEMA_FILE)) + }.to change { read_artifact(PROTO_SCHEMA_FILE) } + .from(nil) + .to( + start_with( + "// Generated by `bundle exec rake schema_artifacts:dump`.\n" \ + "// DO NOT EDIT BY HAND. Any edits will be lost the next time the rake task is run.\n" + ).and(a_string_including('syntax = "proto3";', "message Product", "string name = 2;")) + ) + end + + it "idempotently dumps proto artifacts" do + write_proto_schema(table_defs: <<~EOS) + s.object_type "Product" do |t| + t.field "id", "ID" + t.index "products" + end + EOS + + run_rake_with_proto("schema_artifacts:dump") + + expect { + output = run_rake_with_proto("schema_artifacts:dump") + expect(output.lines).to include(a_string_including("already up to date", PROTO_SCHEMA_FILE)) + }.to maintain { read_artifact(PROTO_SCHEMA_FILE) } + end + + it "can persist and reuse proto field-number mappings from an artifact file" do + write_proto_schema(table_defs: <<~EOS) + s.object_type "Product" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "products" + end + EOS + + run_rake_with_proto("schema_artifacts:dump") + + expect(read_artifact(PROTO_FIELD_NUMBERS_FILE)).not_to be_nil + expect(parsed_proto_field_numbers).to eq({ + "enums" => {}, + "messages" => { + "Product" => { + "fields" => { + "id" => 1, + "name" => 2 + } + } + } + }) + + write_proto_schema(table_defs: <<~EOS) + s.object_type "Product" do |t| + t.field "name", "String" + t.field "id", "ID" + t.index "products" + end + EOS + + run_rake_with_proto("schema_artifacts:dump") + + expect(read_artifact(PROTO_SCHEMA_FILE)).to include("string name = 2;") + expect(read_artifact(PROTO_SCHEMA_FILE)).to include("string id = 1;") + end + end + + private + + def write_proto_schema(table_defs:) + ::File.write("schema.rb", <<~EOS) + ElasticGraph.define_schema do |s| + #{table_defs} + end + EOS + end + + def run_rake_with_proto(*args) + run_rake(*args) do |output| + ElasticGraph::SchemaDefinition::RakeTasks.new( + schema_element_name_form: :snake_case, + index_document_sizes: false, + path_to_schema: "schema.rb", + schema_artifacts_directory: "config/schema/artifacts", + extension_modules: [SchemaDefinition::APIExtension], + output: output + ) + end + end + + def read_artifact(name) + path = File.join("config", "schema", "artifacts", name) + File.read(path) if File.exist?(path) + end + + def parsed_proto_field_numbers + ::YAML.safe_load(read_artifact(PROTO_FIELD_NUMBERS_FILE)) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/spec_helper.rb b/elasticgraph-proto_ingestion/spec/spec_helper.rb index 4e6c40a9a..39286580a 100644 --- a/elasticgraph-proto_ingestion/spec/spec_helper.rb +++ b/elasticgraph-proto_ingestion/spec/spec_helper.rb @@ -8,3 +8,13 @@ # This file contains RSpec configuration for `elasticgraph-proto_ingestion`. # It is loaded by the shared spec helper at `spec_support/spec_helper.rb`. + +RSpec.configure do |config| + config.define_derived_metadata(absolute_file_path: %r{/elasticgraph-proto_ingestion/}) do |meta| + meta[:proto_schema] = true + end + + config.when_first_matching_example_defined(:proto_schema) do + require "support/proto_schema_support" + end +end diff --git a/elasticgraph-proto_ingestion/spec/support/proto_schema_support.rb b/elasticgraph-proto_ingestion/spec/support/proto_schema_support.rb new file mode 100644 index 000000000..20aa71487 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/support/proto_schema_support.rb @@ -0,0 +1,61 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/api_extension" +require "elastic_graph/schema_definition/test_support" + +module ElasticGraph + module ProtoIngestion + module SchemaSupport + include ElasticGraph::SchemaDefinition::TestSupport + + def define_proto_schema(**options, &block) + define_proto_schema_results(**options, &block).proto_schema + end + + def define_proto_schema_results(**options, &block) + define_schema( + schema_element_name_form: :snake_case, + extension_modules: [SchemaDefinition::APIExtension], + **options, + &block + ) + end + + def proto_type_def_from(proto, type) + lines = proto.lines + definition_start = /^(?:enum|message) #{Regexp.escape(type)} \{/ + start_indices = lines.each_index.select { |index| definition_start.match?(lines.fetch(index)) } + + if start_indices.size >= 2 + # :nocov: -- only executed when a mistake has been made; causes a failing test. + raise Errors::SchemaError, + "Expected to find 0 or 1 proto type definition for #{type}, but found #{start_indices.size}." + # :nocov: + end + + definition_start_index = start_indices.first + return nil unless definition_start_index + + brace_depth = 0 + result_lines = lines.drop(definition_start_index).each_with_object([]) do |line, collected_lines| + collected_lines << line + structural_content = line.sub(%r{//.*}, "") + brace_depth += structural_content.count("{") - structural_content.count("}") + break collected_lines if brace_depth.zero? + end + + result_lines.join.strip + end + end + + RSpec.configure do |config| + config.include SchemaSupport, :proto_schema + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb index 7f57ae738..47981d30c 100644 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb @@ -7,25 +7,96 @@ # frozen_string_literal: true require "elastic_graph/proto_ingestion/schema_definition/api_extension" -require "elastic_graph/spec_support/schema_definition_helpers" module ElasticGraph module ProtoIngestion module SchemaDefinition RSpec.describe APIExtension do - include_context "SchemaDefinitionHelpers" - - it "can be used as a schema definition extension module, without customizing the schema definition yet" do - with_extension, without_extension = [[APIExtension], []].map do |extension_modules| - define_schema(schema_element_name_form: "snake_case", extension_modules: extension_modules) do |schema| - schema.object_type "Widget" do |t| - t.field "id", "ID!" - t.index "widgets" + it "emits the configured package name" do + proto = define_proto_schema do |s| + s.proto_schema_artifacts package_name: "sales.v1" + + s.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + end + end + + expect(proto).to include("package sales.v1;") + end + + it "maps every built-in scalar to a proto field type" do + built_in_scalar_options = FactoryExtension::BUILT_IN_SCALAR_PROTO_OPTIONS_BY_NAME + field_types = [] + proto = define_proto_schema do |s| + s.object_type "Widget" do |t| + built_in_scalar_options.each_key do |type_name| + t.field type_name.downcase, type_name end - end.graphql_schema_string + field_types = t.graphql_fields_by_name.values.map { |field| field.type.name } + t.index "widgets" + end end - expect(with_extension).to eq(without_extension) + expect(field_types).to match_array(built_in_scalar_options.keys) + expect(proto).to include('import "google/protobuf/timestamp.proto";') + built_in_scalar_options.each.with_index(1) do |(type_name, options), field_number| + field_name = Identifier.field_name(type_name.downcase) + expect(proto).to include("#{options.fetch(:type)} #{field_name} = #{field_number};") + end + end + + it "requires `package_name` to be a non-empty String" do + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: "" + end + }.to raise_error(Errors::SchemaError, a_string_including("`package_name` must be a non-empty String")) + + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: :symbol_package + end + }.to raise_error(Errors::SchemaError, a_string_including("`package_name` must be a non-empty String")) + end + + it "requires `syntax` to be a supported protobuf syntax" do + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: "elasticgraph", syntax: :proto1 + end + }.to raise_error(Errors::SchemaError, a_string_including("`syntax` must be one of")) + end + + it "requires `headers` to be an Array of Strings" do + expect { + define_proto_schema do |s| + s.proto_schema_artifacts( + package_name: "elasticgraph", + headers: %(option java_package = "com.example";) + ) + end + }.to raise_error(Errors::SchemaError, a_string_including("`headers` must be an Array of Strings")) + + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: "elasticgraph", headers: [:not_a_string] + end + }.to raise_error(Errors::SchemaError, a_string_including("`headers` must be an Array of Strings")) + + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: "elasticgraph", headers: ["valid", :not_a_string] + end + }.to raise_error(Errors::SchemaError, a_string_including("`headers` must be an Array of Strings")) + end + + it "rejects invalid package names when they are configured" do + expect { + define_proto_schema do |s| + s.proto_schema_artifacts package_name: "my-app.events" + end + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`", "my-app.events")) end end end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/factory_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/factory_extension_spec.rb new file mode 100644 index 000000000..1b10b5fc1 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/factory_extension_spec.rb @@ -0,0 +1,60 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/factory_extension" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe FactoryExtension do + it "supports public type definitions without configuration blocks" do + proto = define_proto_schema do |s| + s.enum_type "Status" + s.state.enum_types_by_name.fetch("Status").value "ACTIVE" + + s.interface_type "Named" + s.state.object_types_by_name.fetch("Named").field "name", "String" + + s.object_type "UnconfiguredRecord" + s.state.object_types_by_name.fetch("UnconfiguredRecord").field "id", "ID" + + s.union_type "Entity" + s.state.object_types_by_name.fetch("Entity").subtype "UnconfiguredRecord" + end + + expect(proto).to eq("") + end + + it "validates scalar type definitions without configuration blocks" do + expect { + define_proto_schema do |s| + s.scalar_type "UnconfiguredScalar" + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Protobuf type not configured for scalar type `UnconfiguredScalar`" + )) + end + + it "rejects colliding proto type names as soon as the second type is defined" do + expect { + define_proto_schema do |s| + s.enum_type "package" do |t| + t.value "ACTIVE" + end + + s.object_type "package_" do |t| + t.field "id", "ID" + t.index "packages" + end + end + }.to raise_error(Errors::SchemaError, a_string_including("both map to the same proto type name `package_`")) + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/identifier_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/identifier_spec.rb new file mode 100644 index 000000000..d68594090 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/identifier_spec.rb @@ -0,0 +1,55 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/identifier" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe Identifier do + it "escapes reserved keywords" do + expect(Identifier.message_name("package")).to eq("package_") + expect(Identifier.message_name("custom")).to eq("custom") + end + + it "allows keywords in package name segments" do + expect(Identifier.validate_package_name("proto.package.v1")).to eq("proto.package.v1") + end + + it "rejects package names with segments that are not valid protobuf identifiers" do + expect { + Identifier.validate_package_name("my-app.events") + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`", "my-app.events")) + + expect { + Identifier.validate_package_name("myapp..events") + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`")) + + expect { + Identifier.validate_package_name("1myapp.events") + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`")) + + expect { + Identifier.validate_package_name("") + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`")) + + expect { + Identifier.validate_package_name("myapp.events.") + }.to raise_error(Errors::SchemaError, a_string_including("`package_name`")) + end + + it "escapes message, enum, field, and enum value names" do + expect(Identifier.message_name("service")).to eq("service_") + expect(Identifier.enum_name("message")).to eq("message_") + expect(Identifier.field_name("string")).to eq("string_") + expect(Identifier.enum_value_name("stream")).to eq("stream_") + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb new file mode 100644 index 000000000..0302e8b65 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb @@ -0,0 +1,97 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension" +require "fileutils" +require "stringio" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe SchemaArtifactManagerExtension, :in_temp_dir do + it "dumps proto artifacts alongside the base artifacts" do + artifact_base_names = artifacts_for(define_indexed_type_schema) + + expect(artifact_base_names).to include(PROTO_SCHEMA_FILE, PROTO_FIELD_NUMBERS_FILE) + end + + it "omits proto artifacts when the schema defines no indexed types" do + results = define_proto_schema_results do |s| + s.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + end + end + + artifact_base_names = artifacts_for(results) + + expect(artifact_base_names).not_to include(PROTO_SCHEMA_FILE, PROTO_FIELD_NUMBERS_FILE) + end + + it "seeds proto generation with the field-number mappings from a previously dumped artifact" do + ::FileUtils.mkdir_p("artifacts") + ::File.write(::File.join("artifacts", PROTO_FIELD_NUMBERS_FILE), <<~YAML) + messages: + Widget: + fields: + id: 7 + YAML + + results = define_indexed_type_schema + artifacts_for(results) + + expect(results.proto_schema).to include("string id = 7;") + end + + it "seeds proto generation with the enum value numbers from a previously dumped artifact" do + ::FileUtils.mkdir_p("artifacts") + ::File.write(::File.join("artifacts", PROTO_FIELD_NUMBERS_FILE), <<~YAML) + enums: + Status: + values: + INACTIVE: 5 + YAML + + results = define_proto_schema_results do |s| + s.enum_type "Status" do |t| + t.values "ACTIVE", "INACTIVE" + end + + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "widgets" + end + end + artifacts_for(results) + + expect(results.proto_schema).to include("STATUS_ACTIVE = 1;", "STATUS_INACTIVE = 5;") + end + + def define_indexed_type_schema + define_proto_schema_results do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.index "widgets" + end + end + end + + def artifacts_for(results) + manager = results.state.api.factory.new_schema_artifact_manager( + schema_definition_results: results, + schema_artifacts_directory: "artifacts", + output: ::StringIO.new + ) + + manager.send(:artifacts_from_schema_def).map { |artifact| ::File.basename(artifact.file_name) } + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb new file mode 100644 index 000000000..320cf2479 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_edge_cases_spec.rb @@ -0,0 +1,500 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/api_extension" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe Schema do + it "returns an empty string when no indexed types are present" do + proto = define_proto_schema do |s| + s.object_type "Point" do |t| + t.field "x", "Float" + t.field "y", "Float" + end + end + + expect(proto).to eq("") + end + + it "raises when enum values map to duplicate proto value names" do + expect { + define_proto_schema do |s| + s.enum_type "Status" do |t| + t.values "option", "OPTION" + end + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Enum `Status` values `option` and `OPTION`", + "duplicate proto enum value name `STATUS_OPTION`" + )) + end + + it "raises when an enum value conflicts with the generated zero value" do + expect { + define_proto_schema do |s| + s.enum_type "Status" do |t| + t.values "ACTIVE", "UNSPECIFIED" + end + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Enum `Status` value `UNSPECIFIED`", + "conflicts with the generated zero value `STATUS_UNSPECIFIED`" + )) + end + + it "raises when two fields collapse to the same proto field name after keyword escaping" do + expect { + define_proto_schema do |s| + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "string", "String" + t.field "string_", "String" + t.index "accounts" + end + end + }.to raise_error(Errors::SchemaError, a_string_including("duplicate proto field names")) + end + + it "renders a shared enum only once when multiple indexed types reference it" do + proto = define_proto_schema do |s| + s.enum_type "Status" do |t| + t.value "ACTIVE" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "accounts" + end + + s.object_type "User" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "users" + end + end + + expect(proto.scan(/^enum Status \{/).size).to eq(1) + expect(proto_type_def_from(proto, "Account")).to include("Status status = 2;") + expect(proto_type_def_from(proto, "User")).to include("Status status = 2;") + end + it "raises on field-number mapping collisions for a message" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Account" => { + "id" => 1, + "name" => 1 + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("field-number mapping collision")) + end + + it "normalizes nil field-number mappings to empty hashes" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings nil + end + + expect(results.proto_schema).to eq("") + expect(results.proto_field_number_mappings).to eq({"enums" => {}, "messages" => {}}) + end + + it "validates field-number mapping input type" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings("bad") + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be a Hash")) + end + + it "validates that `messages` is a hash in field-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => "bad"}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must have a `messages` Hash")) + end + + it "accepts symbol `:messages` key in field-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + messages: { + "Account" => { + "id" => 7 + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 7;") + end + + it "accepts symbol `:fields` and nested symbol keys in field-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + messages: { + "Account" => { + fields: { + "id" => { + field_number: 7, + name_in_index: :account_id + } + } + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID", name_in_index: "account_id" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 7;") + end + + it "validates per-message field-number mapping structure" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => "bad"}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be a Hash")) + end + + it "validates that nested `fields` is a hash in field-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => {"fields" => "bad"}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must contain a `fields` Hash")) + end + + it "validates that mapped field numbers are valid protobuf field tags" do + [0, Schema::MAX_FIELD_NUMBER + 1, 19_000, 19_999].each do |invalid_number| + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => {"id" => invalid_number}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including( + "must be a valid protobuf field number", "excluding the reserved 19000-19999 range", invalid_number.to_s + )) + end + end + + it "accepts maximum protobuf numbers while allowing enum values in the field-reserved range" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => {"Account" => {"id" => Schema::MAX_FIELD_NUMBER}}, + # Enum value numbers have no protobuf-reserved range, so 19000-19999 is fine here. + "enums" => {"Status" => {"values" => { + "ACTIVE" => Schema::MAX_ENUM_VALUE_NUMBER, + "INACTIVE" => 19_005 + }}} + } + ) + + s.enum_type "Status" do |t| + t.values "ACTIVE", "INACTIVE" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "accounts" + end + end + + generated = results.proto_schema + expect(generated).to include("string id = #{Schema::MAX_FIELD_NUMBER};") + expect(generated).to include("STATUS_ACTIVE = #{Schema::MAX_ENUM_VALUE_NUMBER};") + expect(generated).to include("STATUS_INACTIVE = 19005;") + end + + it "skips the protobuf-reserved range when allocating new field numbers" do + legacy_mappings = (1..18_999).to_h { |number| ["legacy_field_#{number}", number] } + + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => legacy_mappings}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 20000;") + end + + it "validates that mapped enum value numbers do not exceed the int32 maximum" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"enums" => {"Status" => {"values" => {"ACTIVE" => Schema::MAX_ENUM_VALUE_NUMBER + 1}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be a positive integer no greater than 2147483647")) + end + + it "validates that mapped field numbers are integers" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => {"id" => "abc"}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be an integer")) + end + + it "accepts integer field numbers given as strings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"messages" => {"Account" => {"id" => "7"}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 7;") + end + + it "accepts symbol `:enums` and `:values` keys and shorthand value hashes in enum value-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + enums: { + "Status" => {values: {"INACTIVE" => 5}}, + "Color" => {"RED" => 3} + } + } + ) + + s.enum_type "Status" do |t| + t.values "ACTIVE", "INACTIVE" + end + + s.enum_type "Color" do |t| + t.values "RED" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.field "color", "Color" + t.index "accounts" + end + end + + generated = results.proto_schema + expect(generated).to include("STATUS_ACTIVE = 1;", "STATUS_INACTIVE = 5;", "COLOR_RED = 3;") + end + + it "validates that `enums` is a hash in enum value-number mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"enums" => "bad"}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must have an `enums` Hash")) + end + + it "validates per-enum value-number mapping structure" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"enums" => {"Status" => "bad"}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must contain a `values` Hash")) + end + + it "validates that mapped enum value numbers are positive integers, since 0 is reserved" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"enums" => {"Status" => {"values" => {"ACTIVE" => 0}}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be a positive integer", "_UNSPECIFIED")) + end + + it "validates that mapped enum value numbers are integers rather than truncating them" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings({"enums" => {"Status" => {"values" => {"ACTIVE" => 1.5}}}}) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must be a positive integer", "1.5")) + end + + it "validates that structured mappings include `field_number`" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"messages" => {"Account" => {"fields" => {"id" => {"name_in_index" => "account_id"}}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID", name_in_index: "account_id" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must include `field_number`")) + end + + it "validates that structured mappings use a String or Symbol `name_in_index`" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"messages" => {"Account" => {"fields" => {"id" => {"field_number" => 7, "name_in_index" => 123}}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("must use a String or Symbol `name_in_index`")) + end + + it "defaults structured mappings without `name_in_index` to the field name" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"messages" => {"Account" => {"fields" => {"display_name" => {"field_number" => 7}}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "display_name", "String" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string display_name = 7;") + end + + it "allocates the next available field number when a renamed field has no old mapping entry" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"messages" => {"Account" => {"fields" => {"other_name" => 7}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "display_name", "String" do |f| + f.renamed_from "full_name" + end + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 1;") + expect(results.proto_schema).to include("string display_name = 2;") + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb new file mode 100644 index 000000000..610f91387 --- /dev/null +++ b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/schema_spec.rb @@ -0,0 +1,786 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/proto_ingestion/schema_definition/api_extension" + +module ElasticGraph + module ProtoIngestion + module SchemaDefinition + RSpec.describe Schema do + it "generates a proto schema from indexed types" do + proto = define_proto_schema do |s| + s.enum_type "Status" do |t| + t.documentation "The status of an account.\n\nUsed when ingesting accounts." + t.value "ACTIVE" do |v| + v.documentation "The account is active." + end + t.value "INACTIVE" + end + + s.object_type "Address" do |t| + t.field "street", "String" + t.field "city", "String" + end + + s.object_type "Account" do |t| + t.documentation "An account in the system.\n\n" + t.field "id", "ID" do |f| + f.documentation "The account's unique identifier." + end + t.field "status", "Status" + t.field "address", "Address" + t.field "tags", "[String!]!" + t.field "display_name", "String", graphql_only: true + t.index "accounts" + end + end + + expect(proto).to eq(<<~PROTO) + syntax = "proto3"; + + package elasticgraph; + + // The status of an account. + // + // Used when ingesting accounts. + enum Status { + // The default value when no enum value has been explicitly set. Do not use this value. + // See https://protobuf.dev/programming-guides/proto3/#enum-default. + STATUS_UNSPECIFIED = 0; + // The account is active. + STATUS_ACTIVE = 1; + STATUS_INACTIVE = 2; + } + + // An account in the system. + message Account { + // The account's unique identifier. + string id = 1; + Status status = 2; + Address address = 3; + repeated string tags = 4; + } + + message Address { + string street = 1; + string city = 2; + } + PROTO + end + + it "emits proto2 syntax with an explicit label on every field when `syntax: :proto2`" do + proto = define_proto_schema do |s| + s.proto_schema_artifacts package_name: "elasticgraph", syntax: :proto2 + + s.enum_type "Status" do |t| + t.values "ACTIVE", "INACTIVE" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.field "tags", "[String!]!" + t.index "accounts" + end + end + + expect(proto).to start_with(%(syntax = "proto2";)) + expect(proto_type_def_from(proto, "Account")).to include( + "optional string id = 1;", + "optional Status status = 2;", + "repeated string tags = 3;" + ) + end + + it "renders custom `headers` verbatim after the package declaration" do + proto = define_proto_schema do |s| + s.proto_schema_artifacts( + package_name: "myapp.events.v1", + headers: [ + %(option java_package = "com.myapp.events";), + "option java_multiple_files = true;" + ] + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect(proto).to include(<<~PROTO) + package myapp.events.v1; + + option java_package = "com.myapp.events"; + option java_multiple_files = true; + + message Account { + PROTO + end + + it "generates oneof wrappers for indexed interface and union types" do + proto = define_proto_schema do |s| + s.object_type "Car" do |t| + t.implements "Vehicle" + t.field "id", "ID" + t.field "doors", "Int" + end + + s.object_type "Bike" do |t| + t.implements "Vehicle" + t.field "id", "ID" + t.field "gears", "Int" + end + + s.interface_type "Vehicle" do |t| + t.field "id", "ID" + t.index "vehicles" + end + + s.object_type "Person" do |t| + t.field "id", "ID" + t.field "name", "String" + end + + s.object_type "Company" do |t| + t.field "id", "ID" + t.field "stock_ticker", "String" + end + + s.union_type "Inventor" do |t| + t.subtypes "Person", "Company" + t.index "inventors" + end + end + + expect(proto_type_def_from(proto, "Vehicle")).to eq(<<~PROTO.strip) + message Vehicle { + oneof value { + Car car = 1; + Bike bike = 2; + } + } + PROTO + expect(proto_type_def_from(proto, "Inventor")).to eq(<<~PROTO.strip) + message Inventor { + oneof value { + Person person = 1; + Company company = 2; + } + } + PROTO + expect(proto_type_def_from(proto, "Car")).to include("string id = 1;", "int32 doors = 2;") + expect(proto_type_def_from(proto, "Bike")).to include("string id = 1;", "int32 gears = 2;") + expect(proto_type_def_from(proto, "Person")).to include("string id = 1;", "string name = 2;") + expect(proto_type_def_from(proto, "Company")).to include("string id = 1;", "string stock_ticker = 2;") + expect(proto_type_def_from(proto, "Missing")).to be_nil + expect(proto).not_to include("__typename") + end + + it "flattens nested interfaces and snake-cases multiword oneof alternatives" do + proto = define_proto_schema do |s| + s.object_type "DeliveryVehicle" do |t| + t.implements "MotorVehicle" + t.field "id", "ID" + end + + s.interface_type "MotorVehicle" do |t| + t.implements "Vehicle" + t.field "id", "ID" + end + + s.interface_type "Vehicle" do |t| + t.field "id", "ID" + t.index "vehicles" + end + end + + expect(proto_type_def_from(proto, "Vehicle")).to eq(<<~PROTO.strip) + message Vehicle { + oneof value { + DeliveryVehicle delivery_vehicle = 1; + } + } + PROTO + expect(proto_type_def_from(proto, "DeliveryVehicle")).to include("string id = 1;") + expect(proto_type_def_from(proto, "MotorVehicle")).to be_nil + end + + it "rejects lists of lists" do + expect { + define_proto_schema do |s| + s.object_type "Matrix" do |t| + t.field "id", "ID" + t.field "values", "[[Float!]!]!" + t.index "matrices" + end + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Field `Matrix.values` has type `[[Float!]!]!`", + "Protocol Buffers cannot represent lists of lists directly", + "at most one list level" + )) + end + + it "uses custom proto scalar mappings" do + proto = define_proto_schema do |s| + s.scalar_type "CustomTimestamp" do |t| + t.mapping type: "date" + t.protobuf type: "int64" + end + + s.object_type "Event" do |t| + t.field "id", "ID" + t.field "occurred_at", "CustomTimestamp" + t.index "events" + end + end + + expect(proto_type_def_from(proto, "Event")).to include("int64 occurred_at = 2;") + end + + it "maps `DateTime` fields to `google.protobuf.Timestamp`, importing its proto file once" do + proto = define_proto_schema do |s| + s.object_type "Event" do |t| + t.field "id", "ID" + t.field "created_at", "DateTime" + t.field "updated_at", "DateTime" + t.index "events" + end + end + + expect(proto).to eq(<<~PROTO) + syntax = "proto3"; + + package elasticgraph; + + import "google/protobuf/timestamp.proto"; + + message Event { + string id = 1; + google.protobuf.Timestamp created_at = 2; + google.protobuf.Timestamp updated_at = 3; + } + PROTO + end + + it "sorts and de-duplicates imports shared by distinct protobuf types" do + proto = define_proto_schema do |s| + s.scalar_type "FirstZType" do |t| + t.mapping type: "keyword" + t.protobuf type: "example.FirstZType", import: "z/types.proto" + end + + s.scalar_type "SecondZType" do |t| + t.mapping type: "keyword" + t.protobuf type: "example.SecondZType", import: "z/types.proto" + end + + s.scalar_type "AType" do |t| + t.mapping type: "keyword" + t.protobuf type: "example.AType", import: "a/types.proto" + end + + s.object_type "Event" do |t| + t.field "id", "ID" + t.field "first_z", "FirstZType" + t.field "second_z", "SecondZType" + t.field "a", "AType" + t.index "events" + end + end + + expect(proto.lines.grep(/\Aimport /).map(&:chomp)).to eq([ + %(import "a/types.proto";), + %(import "z/types.proto";) + ]) + end + + it "renders format comments on fields whose scalar type documents one" do + proto = define_proto_schema do |s| + s.object_type "Person" do |t| + t.field "id", "ID" + t.field "birth_date", "Date" + t.index "people" + end + end + + expect(proto).to include( + %(string birth_date = 2; // ISO 8601 date, e.g. "2024-11-25") + ) + end + + it "combines source name and format comments on a single field" do + proto = define_proto_schema do |s| + s.object_type "Person" do |t| + t.field "id", "ID" + t.field "option", "Date" + t.index "people" + end + end + + expect(proto).to include( + %(string option_ = 2; // source name: option; ISO 8601 date, e.g. "2024-11-25") + ) + end + + it "supports `import:` and `comment:` on custom scalar types" do + proto = define_proto_schema do |s| + s.scalar_type "Money" do |t| + t.mapping type: "keyword" + t.protobuf type: "myapp.types.Money", import: "myapp/types/money.proto", comment: "amount + currency" + end + + s.object_type "Order" do |t| + t.field "id", "ID" + t.field "total", "Money" + t.index "orders" + end + end + + expect(proto).to include('import "myapp/types/money.proto";') + expect(proto).to include("myapp.types.Money total = 2; // amount + currency") + end + + it "can assign field numbers from configured mappings" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Account" => { + "id" => 10, + "name" => 2 + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "accounts" + end + end + + generated = results.proto_schema + expect(generated).to include("string id = 10;") + expect(generated).to include("string name = 2;") + end + + it "assigns new field numbers after mapped values when mappings are partial" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Account" => { + "id" => 1 + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string name = 2;") + end + + it "exposes generated field-number mappings as an artifact hash" do + results = define_proto_schema_results do |s| + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "accounts" + end + end + + expect(results.proto_field_number_mappings).to eq({ + "enums" => {}, + "messages" => { + "Account" => { + "fields" => { + "id" => 1, + "name" => 2 + } + } + } + }) + end + + it "exposes field and enum number mappings in canonical order" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "ZMessage" => {"fields" => {"second" => 2, "first" => 1}}, + "AMessage" => {"fields" => {"only" => 3}} + }, + "enums" => { + "ZEnum" => {"values" => {"SECOND" => 2, "FIRST" => 1}}, + "AEnum" => {"values" => {"ONLY" => 3}} + } + } + ) + end + + mappings = results.proto_field_number_mappings + expect(mappings.fetch("messages").keys).to eq(["AMessage", "ZMessage"]) + expect(mappings.dig("messages", "ZMessage", "fields").keys).to eq(["first", "second"]) + expect(mappings.fetch("enums").keys).to eq(["AEnum", "ZEnum"]) + expect(mappings.dig("enums", "ZEnum", "values").keys).to eq(["FIRST", "SECOND"]) + end + + it "preserves reserved numbers for removed fields and allocates new numbers above them" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Account" => { + "id" => 1, + "legacyField" => 2 + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "name", "String" + t.index "accounts" + end + end + + generated = results.proto_schema + expect(generated).to include("string id = 1;") + expect(generated).to include("string name = 3;") + + expect(results.proto_field_number_mappings).to eq({ + "enums" => {}, + "messages" => { + "Account" => { + "fields" => { + "id" => 1, + "legacyField" => 2, + "name" => 3 + } + } + } + }) + end + + it "uses public field names in schema.proto and stores name_in_index overrides in the mapping artifact" do + results = define_proto_schema_results do |s| + s.object_type "Widget" do |t| + t.field "id", "ID" + t.field "display_name", "String", name_in_index: "displayName" + t.index "widgets" + end + end + + expect(results.proto_schema).to include("string display_name = 2;") + expect(results.proto_schema).not_to include("displayName") + + expect(results.proto_field_number_mappings).to eq({ + "enums" => {}, + "messages" => { + "Widget" => { + "fields" => { + "id" => 1, + "display_name" => { + "field_number" => 2, + "name_in_index" => "displayName" + } + } + } + } + }) + end + + it "preserves a field number across a public field rename" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Account" => { + "fields" => { + "full_name" => 7 + } + } + } + } + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "display_name", "String" do |f| + f.renamed_from "full_name" + end + t.index "accounts" + end + end + + expect(results.proto_schema).to include("string id = 1;") + expect(results.proto_schema).to include("string display_name = 7;") + expect(results.proto_field_number_mappings).to eq({ + "enums" => {}, + "messages" => { + "Account" => { + "fields" => { + "id" => 1, + "display_name" => 7 + } + } + } + }) + end + + it "generates the proto schema and assigns field numbers when no renames are declared" do + results = define_proto_schema_results do |s| + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "display_name", "String" + t.index "accounts" + end + end + + expect(results.proto_schema).to eq(<<~PROTO) + syntax = "proto3"; + + package elasticgraph; + + message Account { + string id = 1; + string display_name = 2; + } + PROTO + + expect(results.proto_field_number_mappings).to eq({ + "enums" => {}, + "messages" => { + "Account" => { + "fields" => { + "id" => 1, + "display_name" => 2 + } + } + } + }) + end + + it "preserves enum value numbers from configured mappings, reserving removed values' numbers" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "enums" => { + "Status" => { + "values" => { + "ACTIVE" => 1, + "PAUSED" => 2, + "INACTIVE" => 3 + } + } + } + } + ) + + # `PAUSED` has been removed since the mappings were dumped, and `ARCHIVED` has been added. + s.enum_type "Status" do |t| + t.values "ACTIVE", "INACTIVE", "ARCHIVED" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "accounts" + end + end + + expect(proto_type_def_from(results.proto_schema, "Status")).to include( + "STATUS_ACTIVE = 1;", + "STATUS_INACTIVE = 3;", + "STATUS_ARCHIVED = 4;" + ) + + # `PAUSED` keeps its number reserved in the artifact so it is never reused for a new value. + expect(results.proto_field_number_mappings.fetch("enums")).to eq({ + "Status" => { + "values" => { + "ACTIVE" => 1, + "PAUSED" => 2, + "INACTIVE" => 3, + "ARCHIVED" => 4 + } + } + }) + end + + it "raises a clear error when the configured mappings assign the same number to two enum values" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"enums" => {"Status" => {"values" => {"ACTIVE" => 1, "INACTIVE" => 1}}}} + ) + + s.enum_type "Status" do |t| + t.values "ACTIVE", "INACTIVE" + end + + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "status", "Status" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including( + "enum value-number mapping collision in enum `Status`", + "`ACTIVE` and `INACTIVE`", + "number 1" + )) + end + + it "rejects field numbers that are not integers rather than silently truncating them" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + {"messages" => {"Account" => {"fields" => {"id" => 1.5}}}} + ) + + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + expect { + results.proto_schema + }.to raise_error(Errors::SchemaError, a_string_including("`Account.id`", "must be an integer", "1.5")) + end + + it "preserves stable field numbers for oneof alternatives" do + results = define_proto_schema_results do |s| + s.configure_proto_field_number_mappings( + { + "messages" => { + "Vehicle" => { + "fields" => { + "removed" => 1, + "car" => 4, + "bike" => 8 + } + } + } + } + ) + + ["Car", "Bike", "Scooter"].each do |type_name| + s.object_type type_name do |t| + t.field "id", "ID" + end + end + + s.union_type "Vehicle" do |t| + t.subtypes "Car", "Bike", "Scooter" + t.index "vehicles" + end + end + + vehicle = proto_type_def_from(results.proto_schema, "Vehicle") + expect(vehicle).to include("Car car = 4;", "Bike bike = 8;", "Scooter scooter = 2;") + expect(results.proto_field_number_mappings.fetch("messages").fetch("Vehicle")).to eq({ + "fields" => { + "removed" => 1, + "scooter" => 2, + "car" => 4, + "bike" => 8 + } + }) + end + + it "renders independently each time it is called" do + results = define_proto_schema_results do |s| + s.object_type "Account" do |t| + t.field "id", "ID" + t.index "accounts" + end + end + + generator = Schema.new( + state: results.state, + all_types: results.send(:all_types), + package_name: "elasticgraph" + ) + + first_generation = generator.to_proto + + second_generation = generator.to_proto + expect(second_generation).to eq(first_generation) + expect(second_generation).not_to be(first_generation) + end + + it "raises when a custom scalar is defined without a protobuf type" do + expect { + define_proto_schema do |s| + s.scalar_type "UnconfiguredScalar" do |t| + t.mapping type: "keyword" + end + end + }.to raise_error(Errors::SchemaError, a_string_including( + "Protobuf type not configured for scalar type `UnconfiguredScalar`.", + "call `protobuf type:" + )) + end + + it "does not require a protobuf type for GraphQL-only scalars" do + proto = define_proto_schema do |s| + s.scalar_type "GraphQLOnlyScalar" do |t| + t.mapping type: "keyword" + t.graphql_only true + end + end + + expect(proto).to eq("") + end + + it "resolves proto field types for built-in scalars that are renamed via `type_name_overrides`" do + proto = define_proto_schema(type_name_overrides: {"JsonSafeLong" => "BigNumber"}) do |s| + s.object_type "Account" do |t| + t.field "id", "ID" + t.field "count", "BigNumber" + t.index "accounts" + end + end + + expect(proto_type_def_from(proto, "Account")).to include("int64 count = 2;") + end + + it "prefixes enum values and escapes proto keywords in generated identifiers" do + proto = define_proto_schema do |s| + s.enum_type "Command" do |t| + t.values "option", "stream" + end + + s.object_type "Request" do |t| + t.field "id", "ID" + t.field "package", "String" + t.field "command", "Command" + t.index "requests" + end + end + + expect(proto_type_def_from(proto, "Command")).to include("COMMAND_OPTION = 1;", "COMMAND_STREAM = 2;") + expect(proto_type_def_from(proto, "Request")).to include("string package_ = 2; // source name: package") + end + end + end + end +end diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb deleted file mode 100644 index 558b42477..000000000 --- a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2024 - 2026 Block, Inc. -# -# Use of this source code is governed by an MIT-style -# license that can be found in the LICENSE file or at -# https://opensource.org/licenses/MIT. -# -# frozen_string_literal: true - -require "elastic_graph/proto_ingestion" - -module ElasticGraph - RSpec.describe ProtoIngestion do - it "defines the PROTO_SCHEMA_FILE constant" do - expect(ProtoIngestion::PROTO_SCHEMA_FILE).to eq("schema.proto") - end - - it "defines the PROTO_FIELD_NUMBERS_FILE constant" do - expect(ProtoIngestion::PROTO_FIELD_NUMBERS_FILE).to eq("proto_field_numbers.yaml") - end - end -end