diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java b/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java index 625e9fd9d3..5c7fcc59f0 100644 --- a/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java +++ b/parquet-column/src/main/java/org/apache/parquet/schema/LogicalTypeAnnotation.java @@ -188,6 +188,12 @@ protected LogicalTypeAnnotation fromString(List params) { protected LogicalTypeAnnotation fromString(List params) { return unknownType(); } + }, + FILE { + @Override + protected LogicalTypeAnnotation fromString(List params) { + return fileType(); + } }; protected abstract LogicalTypeAnnotation fromString(List params); @@ -378,6 +384,10 @@ public static UnknownLogicalTypeAnnotation unknownType() { return UnknownLogicalTypeAnnotation.INSTANCE; } + public static FileLogicalTypeAnnotation fileType() { + return FileLogicalTypeAnnotation.INSTANCE; + } + public static class StringLogicalTypeAnnotation extends LogicalTypeAnnotation { private static final StringLogicalTypeAnnotation INSTANCE = new StringLogicalTypeAnnotation(); @@ -1229,6 +1239,93 @@ public boolean equals(Object obj) { } } + /** + * File logical type annotation. Annotates a group (struct) that represents a reference to a + * range of bytes, which may be stored inline in the value or in an external file. Every field is + * optional, both in the schema (a writer may omit any field from the group definition) and in the + * data (any field that is present has a field repetition type of {@code OPTIONAL}). Fields are + * identified by name (case sensitively), not by field order. A group need only define the fields + * it uses. The group may contain the following fields: + *
    + *
  • {@code uri} (STRING): a URI-reference (RFC 3986) that identifies an external file, for + * example {@code s3://bucket/file.jpg}.
    • + *
  • {@code offset} (INT64): start of the byte range within the external file identified by + * {@code uri}; if not set, treated as 0. Must not be negative and may only be set together + * with {@code uri}.
    • + *
  • {@code size} (INT64): byte length of the referenced data within the external file. May be + * omitted for a whole-file external reference, in which case the range runs to the end of + * the referenced file. Must be set whenever {@code offset} is set. Must not be negative.
    • + *
  • {@code content_type} (STRING): the media (MIME) type (RFC 2046) of the resolved bytes; + * when not set, {@code application/octet-stream} is assumed.
    • + *
  • {@code checksum} (STRING): a self-describing integrity token for the resolved bytes, of + * the form {@code :}.
    • + *
  • {@code inline} (BYTE_ARRAY): the referenced bytes stored inline in the value.
    • + *
+ * No fields with names other than the above are permitted. The schema builder additionally + * rejects group definitions that could never produce a valid value: a group must declare at least + * one of {@code inline} or {@code uri} (a value resolves to bytes only via inline storage or an + * external reference, so a group declaring neither — even if it declares {@code offset} or + * {@code size} — can never produce a resolvable value), and a group that declares {@code offset} + * must also declare {@code uri} and {@code size} ({@code offset} locates a range within the + * external file identified by {@code uri} and is meaningless without it, and always bounds a range + * that requires {@code size}). Each declared field must also match its required physical type. + * Per-value rules that depend on the data in each row — {@code size} being set whenever + * {@code offset} is set, and {@code offset}/{@code size} being non-negative — cannot be enforced + * here and are the responsibility of writers and consumers. + */ + public static class FileLogicalTypeAnnotation extends LogicalTypeAnnotation { + private static final FileLogicalTypeAnnotation INSTANCE = new FileLogicalTypeAnnotation(); + + /** Field name holding the URI-reference of an external file. */ + public static final String URI_FIELD = "uri"; + + /** Field name holding the start of the byte range. */ + public static final String OFFSET_FIELD = "offset"; + + /** Field name holding the byte length of the referenced data. */ + public static final String SIZE_FIELD = "size"; + + /** Field name holding the media (MIME) type of the resolved bytes. */ + public static final String CONTENT_TYPE_FIELD = "content_type"; + + /** Field name holding the integrity token for the resolved bytes. */ + public static final String CHECKSUM_FIELD = "checksum"; + + /** Field name holding the referenced bytes stored inline. */ + public static final String INLINE_FIELD = "inline"; + + /** All recognized field names in a FILE-annotated group. All fields are optional. */ + public static final Set FIELD_NAMES = + Set.of(URI_FIELD, OFFSET_FIELD, SIZE_FIELD, CONTENT_TYPE_FIELD, CHECKSUM_FIELD, INLINE_FIELD); + + private FileLogicalTypeAnnotation() {} + + @Override + public OriginalType toOriginalType() { + return null; + } + + @Override + public Optional accept(LogicalTypeAnnotationVisitor logicalTypeAnnotationVisitor) { + return logicalTypeAnnotationVisitor.visit(this); + } + + @Override + LogicalTypeToken getType() { + return LogicalTypeToken.FILE; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof FileLogicalTypeAnnotation; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + } + public static class GeometryLogicalTypeAnnotation extends LogicalTypeAnnotation { private final String crs; @@ -1434,5 +1531,9 @@ default Optional visit(GeographyLogicalTypeAnnotation geographyLogicalType) { default Optional visit(UnknownLogicalTypeAnnotation unknownLogicalTypeAnnotation) { return empty(); } + + default Optional visit(FileLogicalTypeAnnotation fileLogicalType) { + return empty(); + } } } diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java index 2f12991ab0..43d2e326c6 100644 --- a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java +++ b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java @@ -821,12 +821,115 @@ public THIS addFields(Type... types) { @Override protected GroupType build(String name) { if (newLogicalTypeSet) { + if (logicalTypeAnnotation instanceof LogicalTypeAnnotation.FileLogicalTypeAnnotation) { + validateFileTypeFields(name, fields); + } return new GroupType(repetition, name, logicalTypeAnnotation, fields, id); } else { return new GroupType(repetition, name, getOriginalType(), fields, id); } } + private static void validateFileTypeFields(String name, List fields) { + boolean hasUri = false; + boolean hasOffset = false; + boolean hasSize = false; + boolean hasInline = false; + for (Type field : fields) { + String fieldName = field.getName(); + if (!LogicalTypeAnnotation.FileLogicalTypeAnnotation.FIELD_NAMES.contains(fieldName)) { + throw new IllegalArgumentException("FILE type group '" + name + "' contains unrecognized field '" + + fieldName + "'. Valid fields are: " + + String.join(", ", LogicalTypeAnnotation.FileLogicalTypeAnnotation.FIELD_NAMES)); + } + Preconditions.checkArgument( + field.isPrimitive() && field.getRepetition() == Type.Repetition.OPTIONAL, + "FILE type field '%s' must be an optional primitive in group '%s'", + fieldName, + name); + validateFileTypeFieldPhysicalType(name, field.asPrimitiveType()); + if (LogicalTypeAnnotation.FileLogicalTypeAnnotation.URI_FIELD.equals(fieldName)) { + hasUri = true; + } else if (LogicalTypeAnnotation.FileLogicalTypeAnnotation.OFFSET_FIELD.equals(fieldName)) { + hasOffset = true; + } else if (LogicalTypeAnnotation.FileLogicalTypeAnnotation.SIZE_FIELD.equals(fieldName)) { + hasSize = true; + } else if (LogicalTypeAnnotation.FileLogicalTypeAnnotation.INLINE_FIELD.equals(fieldName)) { + hasInline = true; + } + } + // A value resolves to bytes only via inline storage or an external reference, so a group must + // declare at least one of `inline` or `uri`. A group declaring neither — even if it declares + // `offset` or `size` — can never produce a resolvable value, so reject it at schema-build + // time. + Preconditions.checkArgument( + hasInline || hasUri, + "FILE type group '%s' must declare at least one of 'inline' or 'uri'; a value resolves to " + + "bytes only via inline storage or an external reference, so a group declaring " + + "neither can never produce a valid value", + name); + // `offset` locates a byte range within the external file identified by `uri`, so it is + // meaningless without `uri`. A group that declares `offset` but not `uri` can never produce a + // valid value, so reject it at schema-build time. + Preconditions.checkArgument( + !hasOffset || hasUri, + "FILE type group '%s' declares field 'offset' but not 'uri'; 'offset' locates a range " + + "within an external file and may only be set together with 'uri'", + name); + // The spec requires `size` to be set whenever `offset` is set. A group that declares + // `offset` but not `size` can never produce a valid value, so reject it at schema-build + // time. + Preconditions.checkArgument( + !hasOffset || hasSize, + "FILE type group '%s' declares field 'offset' but not 'size'; 'size' is required whenever 'offset' is set", + name); + // The remaining spec rules are per-value constraints the schema builder cannot verify because + // it sees only which fields are declared, not their values in each row: `size` being set + // whenever `offset` is set, and `offset`/`size` being non-negative. Those are the + // responsibility of writers and consumers of FILE values. + } + + /** + * Validates that a declared FILE field uses the physical type required by the spec: + * {@code uri}, {@code content_type}, and {@code checksum} are STRING (BINARY), {@code offset} + * and {@code size} are INT64, and {@code inline} is BYTE_ARRAY (BINARY). + */ + private static void validateFileTypeFieldPhysicalType(String name, PrimitiveType field) { + String fieldName = field.getName(); + PrimitiveType.PrimitiveTypeName physicalType = field.getPrimitiveTypeName(); + switch (fieldName) { + case LogicalTypeAnnotation.FileLogicalTypeAnnotation.URI_FIELD: + case LogicalTypeAnnotation.FileLogicalTypeAnnotation.CONTENT_TYPE_FIELD: + case LogicalTypeAnnotation.FileLogicalTypeAnnotation.CHECKSUM_FIELD: + Preconditions.checkArgument( + physicalType == PrimitiveType.PrimitiveTypeName.BINARY + && field.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.StringLogicalTypeAnnotation, + "FILE type field '%s' must be a STRING (BINARY annotated as STRING) in group '%s'", + fieldName, + name); + break; + case LogicalTypeAnnotation.FileLogicalTypeAnnotation.OFFSET_FIELD: + case LogicalTypeAnnotation.FileLogicalTypeAnnotation.SIZE_FIELD: + Preconditions.checkArgument( + physicalType == PrimitiveType.PrimitiveTypeName.INT64, + "FILE type field '%s' must be an INT64 in group '%s'", + fieldName, + name); + break; + case LogicalTypeAnnotation.FileLogicalTypeAnnotation.INLINE_FIELD: + Preconditions.checkArgument( + physicalType == PrimitiveType.PrimitiveTypeName.BINARY, + "FILE type field '%s' must be a BYTE_ARRAY (BINARY) in group '%s'", + fieldName, + name); + break; + default: + // Unreachable: field names are validated against FIELD_NAMES before this call. + break; + } + } + public MapBuilder map(Type.Repetition repetition) { return new MapBuilder<>(self()).repetition(repetition); } diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java index 0d7791a19b..57f0f91d98 100644 --- a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java +++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java @@ -549,4 +549,268 @@ public void testVariantLogicalTypeWithShredded() { assertThat(((LogicalTypeAnnotation.VariantLogicalTypeAnnotation) annotation).getSpecVersion()) .isEqualTo(specVersion); } + + @Test + public void testFileLogicalTypeUriOnly() { + String name = "file_field"; + GroupType file = new GroupType( + REQUIRED, + name, + LogicalTypeAnnotation.fileType(), + Types.optional(BINARY).as(LogicalTypeAnnotation.stringType()).named("uri")); + + assertThat(file.toString()) + .isEqualTo("required group file_field (FILE) {\n" + " optional binary uri (STRING);\n" + "}"); + + LogicalTypeAnnotation annotation = file.getLogicalTypeAnnotation(); + assertThat(annotation.getType()).isEqualTo(LogicalTypeAnnotation.LogicalTypeToken.FILE); + assertThat(annotation.toOriginalType()).isNull(); + assertThat(annotation).isInstanceOf(LogicalTypeAnnotation.FileLogicalTypeAnnotation.class); + } + + @Test + public void testFileLogicalTypeAllFields() { + String name = "file_field"; + GroupType file = Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("uri") + .optional(INT64) + .named("offset") + .optional(INT64) + .named("size") + .optional(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("content_type") + .optional(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("checksum") + .optional(BINARY) + .named("inline") + .named(name); + + LogicalTypeAnnotation annotation = file.getLogicalTypeAnnotation(); + assertThat(annotation).isInstanceOf(LogicalTypeAnnotation.FileLogicalTypeAnnotation.class); + assertThat(file.getFieldCount()).isEqualTo(6); + assertThat(file.getType("uri").getName()).isEqualTo("uri"); + assertThat(file.getType("offset").getName()).isEqualTo("offset"); + assertThat(file.getType("size").getName()).isEqualTo("size"); + assertThat(file.getType("content_type").getName()).isEqualTo("content_type"); + assertThat(file.getType("checksum").getName()).isEqualTo("checksum"); + assertThat(file.getType("inline").getName()).isEqualTo("inline"); + } + + @Test + public void testFileLogicalTypeInlineOnly() { + // Every field is optional, so an inline-only group is valid (spec inline case). + GroupType file = Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(BINARY) + .named("inline") + .named("inline_file"); + + assertThat(file.getLogicalTypeAnnotation()).isInstanceOf(LogicalTypeAnnotation.FileLogicalTypeAnnotation.class); + assertThat(file.getFieldCount()).isEqualTo(1); + assertThat(file.getType("inline").getName()).isEqualTo("inline"); + } + + @Test + public void testFileLogicalTypeOffsetRequiresUri() { + // 'offset' locates a byte range within the external file identified by 'uri', so it is + // meaningless without 'uri'. A group declaring 'offset' (with 'size' and even 'inline') but + // no 'uri' is rejected at build time. + assertThatThrownBy(() -> Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(INT64) + .named("offset") + .optional(INT64) + .named("size") + .optional(BINARY) + .named("inline") + .named("file_offset_without_uri")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testFileLogicalTypeExternalRangedReferenceWithoutInline() { + // An external ranged reference declares 'uri' + 'offset' + 'size' to point at a byte range of + // an external file. Because 'uri' is declared, the schema is treated as an external-reference + // schema and is not required to declare 'inline', even though it declares 'offset'. + GroupType file = Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("uri") + .optional(INT64) + .named("offset") + .optional(INT64) + .named("size") + .named("external_ranged_file"); + + assertThat(file.getLogicalTypeAnnotation()).isInstanceOf(LogicalTypeAnnotation.FileLogicalTypeAnnotation.class); + assertThat(file.getFieldCount()).isEqualTo(3); + } + + @Test + public void testFileLogicalTypeMetadataOnlyRejected() { + // A value resolves to bytes only via 'inline' or 'uri'. A group declaring only metadata fields + // can never produce a resolvable value. + assertThatThrownBy(() -> Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("content_type") + .optional(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("checksum") + .named("file_metadata_only")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testFileLogicalTypeSizeOnlyRejected() { + // 'size' alone never resolves to bytes, so a size-only group is rejected: it declares neither + // 'inline' nor 'uri'. + assertThatThrownBy(() -> Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(INT64) + .named("size") + .named("file_size_only")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testFileLogicalTypeOffsetRequiresSize() { + // The spec requires 'size' whenever 'offset' is set, so a group declaring 'offset' + // without 'size' can never produce a valid value and is rejected at build time. + assertThatThrownBy(() -> Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("uri") + .optional(INT64) + .named("offset") + .named("file_offset_without_size")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testFileLogicalTypeOffsetWithSize() { + // 'offset' accompanied by 'size' is valid. + GroupType file = Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("uri") + .optional(INT64) + .named("offset") + .optional(INT64) + .named("size") + .named("file_offset_with_size"); + + assertThat(file.getLogicalTypeAnnotation()).isInstanceOf(LogicalTypeAnnotation.FileLogicalTypeAnnotation.class); + assertThat(file.getFieldCount()).isEqualTo(3); + } + + @Test + public void testFileLogicalTypeSizeWithoutOffset() { + // 'uri' + 'size' (without 'offset') is valid: an external reference to '[0, size)'. + GroupType file = Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("uri") + .optional(INT64) + .named("size") + .named("file_size_without_offset"); + + assertThat(file.getLogicalTypeAnnotation()).isInstanceOf(LogicalTypeAnnotation.FileLogicalTypeAnnotation.class); + assertThat(file.getFieldCount()).isEqualTo(2); + } + + @Test + public void testFileLogicalTypeRejectsUnrecognizedField() { + assertThatThrownBy(() -> Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("uri") + .optional(BINARY) + .named("unknown_field") + .named("file_with_bad_field")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testFileLogicalTypeRejectsRequiredField() { + // All FILE fields must have OPTIONAL repetition under the current spec. + assertThatThrownBy(() -> Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .required(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("uri") + .named("file_with_required_uri")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testFileLogicalTypeRejectsGroupField() { + // FILE fields must be primitives, not nested groups. + assertThatThrownBy(() -> Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optionalGroup() + .optional(BINARY) + .named("nested") + .named("uri") + .named("file_with_group_field")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testFileLogicalTypeRejectsWrongStringPhysicalType() { + // 'uri' must be a STRING (BINARY annotated as STRING); an INT64 is rejected. + assertThatThrownBy(() -> Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(INT64) + .named("uri") + .named("file_uri_wrong_type")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testFileLogicalTypeRejectsUnannotatedStringField() { + // A STRING field must carry the STRING logical annotation; plain BINARY is rejected. + assertThatThrownBy(() -> Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(BINARY) + .named("uri") + .named("file_uri_unannotated")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testFileLogicalTypeRejectsWrongInt64PhysicalType() { + // 'offset' and 'size' must be INT64; an INT32 is rejected. + assertThatThrownBy(() -> Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("uri") + .optional(INT32) + .named("size") + .named("file_size_wrong_type")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testFileLogicalTypeRejectsWrongInlinePhysicalType() { + // 'inline' must be a BYTE_ARRAY (BINARY); an INT64 is rejected. + assertThatThrownBy(() -> Types.requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(INT64) + .named("inline") + .named("file_inline_wrong_type")) + .isInstanceOf(IllegalArgumentException.class); + } } diff --git a/parquet-format-structures/src/main/java/org/apache/parquet/format/LogicalTypes.java b/parquet-format-structures/src/main/java/org/apache/parquet/format/LogicalTypes.java index 8956d3944e..8aa21e0ae3 100644 --- a/parquet-format-structures/src/main/java/org/apache/parquet/format/LogicalTypes.java +++ b/parquet-format-structures/src/main/java/org/apache/parquet/format/LogicalTypes.java @@ -60,4 +60,5 @@ public static LogicalType VARIANT(byte specificationVersion) { public static final LogicalType BSON = LogicalType.BSON(new BsonType()); public static final LogicalType FLOAT16 = LogicalType.FLOAT16(new Float16Type()); public static final LogicalType UUID = LogicalType.UUID(new UUIDType()); + public static final LogicalType FILE = LogicalType.FILE(new FileType()); } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java index 465516e48f..0df057a53d 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java @@ -591,6 +591,11 @@ public Optional visit(LogicalTypeAnnotation.GeographyLogicalTypeAnn geographyType.setAlgorithm(fromParquetEdgeInterpolationAlgorithm(geographyLogicalType.getAlgorithm())); return of(LogicalType.GEOGRAPHY(geographyType)); } + + @Override + public Optional visit(LogicalTypeAnnotation.FileLogicalTypeAnnotation fileLogicalType) { + return of(LogicalTypes.FILE); + } } private void addRowGroup( @@ -1389,6 +1394,8 @@ LogicalTypeAnnotation getLogicalTypeAnnotation(LogicalType type) { case VARIANT: VariantType variant = type.getVARIANT(); return LogicalTypeAnnotation.variantType(variant.getSpecification_version()); + case FILE: + return LogicalTypeAnnotation.fileType(); default: throw new RuntimeException("Unknown logical type " + type); } diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java index 4d361d6aa0..31d6a7380e 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java @@ -2279,4 +2279,57 @@ public void testColumnIndexNanCountsRoundTrip() { assertThat(roundTrip).isNotNull(); assertThat(roundTrip.getNanCounts()).containsExactly(1L, 0L, 0L); } + + @Test + public void testFileLogicalType() { + ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); + + MessageType expected = Types.buildMessage() + .requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("uri") + .optional(PrimitiveTypeName.INT64) + .named("offset") + .optional(PrimitiveTypeName.INT64) + .named("size") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("content_type") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("checksum") + .optional(PrimitiveTypeName.BINARY) + .named("inline") + .named("f") + .named("example"); + + List parquetSchema = parquetMetadataConverter.toParquetSchema(expected); + MessageType schema = parquetMetadataConverter.fromParquetSchema(parquetSchema, null); + assertThat(schema).isEqualTo(expected); + LogicalTypeAnnotation logicalType = schema.getType("f").getLogicalTypeAnnotation(); + assertThat(logicalType).isInstanceOf(LogicalTypeAnnotation.FileLogicalTypeAnnotation.class); + assertThat(logicalType).isEqualTo(LogicalTypeAnnotation.fileType()); + } + + @Test + public void testFileLogicalTypeRoundTripUriOnly() { + ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter(); + + MessageType expected = Types.buildMessage() + .requiredGroup() + .as(LogicalTypeAnnotation.fileType()) + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("uri") + .named("f") + .named("example"); + + List parquetSchema = parquetMetadataConverter.toParquetSchema(expected); + MessageType schema = parquetMetadataConverter.fromParquetSchema(parquetSchema, null); + assertThat(schema).isEqualTo(expected); + LogicalTypeAnnotation logicalType = schema.getType("f").getLogicalTypeAnnotation(); + assertThat(logicalType).isInstanceOf(LogicalTypeAnnotation.FileLogicalTypeAnnotation.class); + } }