From dd293fbd12e2ca19bfed7c00a3f41d2e7d24a31d Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 5 Oct 2026 16:22:17 -0600 Subject: [PATCH 1/8] Make typed reads and parameter binding statically compilable `juliac --trim=safe` rejected typed queries: an application with a few hundred row structs hit 1,455 verifier errors, all in Postgres.jl. Typed fields that are not timestamps or decimals were decoded by the column's OID decoder, which returns `Any`, so every field's `StructUtils.make` call was a runtime dispatch. Reading a json/jsonb (or any non-text) column into a `String` field also failed with a MethodError, because the OID decoder produced lazy JSON, integers, UUIDs and so on. With the default `PostgresStyle`, fields whose type Postgres.jl parses from text (strings, symbols, enums, integers, floats, Bool, Char, Date, Time, DateTime, UUID, bytea, lazy JSON, timestamps, decimals, and vectors of these, optionally unioned with Nothing or Missing) now decode from the column text by their declared type. The source stays a `String`, so each read compiles to static calls. A `String` field receives any column's text. Decoders keep the conversions that worked before: integer fields accept integral numeric text such as "12.00" and throw InexactError when out of range, and a Date field reads the UTC date of a timestamptz. A parser registered for the column still applies when its declared type fits the field. Custom styles keep the OID decoder, so their lifts still receive decoded values; other field types keep it too. Also fixed while verifying under `--trim`: - Protocol strings containing a NUL byte threw UndefVarError (the check referred to `Postgres.PostgresInterfaceError` from inside the API module) instead of PostgresInterfaceError. - The startup message took an unused `statement_timeout` argument; with `options` passed as a String the call no longer needs runtime dispatch. - `writemessages` and tuple parameters to `execute` iterated heterogeneous tuples in a loop; both now unroll. - Date and DateTime parameters were formatted with `string(x)`, whose Dates printing is not trim-safe; they use an equivalent static formatter. Behavior changes for typed fields read with the default style: an Int or Float field over a bool column now errors (it previously read 1 or 0), and a registered parser whose declared type does not fit the field is no longer applied before converting; the field parses the column text instead. Co-Authored-By: Claude Opus 5.5 --- docs/src/manual.md | 2 ++ src/api/API.jl | 22 ++++++++------ src/api/types.jl | 64 +++++++++++++++++++++++++++++++++++------ src/array_parsing.jl | 10 +++---- src/execute.jl | 19 ++++++++++++ test/result_decoding.jl | 61 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 155 insertions(+), 23 deletions(-) diff --git a/docs/src/manual.md b/docs/src/manual.md index 0124c35..42548d9 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -119,6 +119,8 @@ profiles = DBInterface.execute(conn, """ The `postgres=(name=:column_name,)` tag is only needed when a column should map to a differently named field. Columns such as `id` or `name` can be left untagged because they already match the Julia field name. +With the default `Postgres.PostgresStyle`, a field whose type Postgres.jl can parse from text (strings, symbols, enums, integers, floats, `Bool`, `Char`, `Date`, `Time`, `DateTime`, `UUID`, `Vector{UInt8}`, lazy JSON, timestamps, decimals, and vectors of these) is decoded from the column text by its declared type, optionally wrapped in `Union{Nothing, …}` or `Union{Missing, …}`. A `String` field receives any column's text, so a `json` or `jsonb` column can be read as its JSON text. An integer field also accepts integral `numeric` values such as `12.00`. A parser registered with `register_type!` still applies when it returns the field's type. Typed reads of these fields compile to static calls, so they work in `juliac --trim` builds. Other field types, and all fields read with a custom driver style, receive the value decoded by the column's type, so custom-style `lift` methods see decoded values. + ### Driver Styles Connection behavior such as query logging, server notices, and asynchronous diff --git a/src/api/API.jl b/src/api/API.jl index f4e0332..73279e6 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -229,14 +229,14 @@ _msgsizeof_parts(parts::Tuple) = msgsizeof(first(parts)) + _msgsizeof_parts(Base writepart(io, x) = write(io, x) function writepart(io, x::String) - occursin('\0', x) && throw(Postgres.PostgresInterfaceError("PostgreSQL protocol strings cannot contain a NUL byte")) + occursin('\0', x) && throw(PostgresInterfaceError("PostgreSQL protocol strings cannot contain a NUL byte")) write(io, x) write(io, UInt8(0)) end writepart(io, x::Integer) = write(io, hton(x)) function writepart(io, x::Tuple{String, String}) (occursin('\0', x[1]) || occursin('\0', x[2])) && - throw(Postgres.PostgresInterfaceError("PostgreSQL startup parameters cannot contain a NUL byte")) + throw(PostgresInterfaceError("PostgreSQL startup parameters cannot contain a NUL byte")) write(io, x[1]) write(io, UInt8(0)) write(io, x[2]) @@ -301,15 +301,14 @@ function writestartupmessage( user::String, dbname::String, application_name::Union{Nothing, String}, - options::Union{Nothing, String}, - statement_timeout::Union{Nothing, Int}, + options::String, )::Nothing # statement_timeout is applied with a SET after connect rather than through # the startup `options` parameter: poolers (pgbouncer) reject unknown # startup options outright, so sending it here fails the whole connection. # A caller-supplied `options` value is different: it is passed through as # given, like libpq does, and an empty value is not sent at all. - send_options = options !== nothing && !isempty(options) + send_options = !isempty(options) len = 8 + msgsizeof(("user", user)) + msgsizeof(("database", dbname)) + msgsizeof(("client_encoding", "UTF8")) + 1 application_name !== nothing && (len += msgsizeof(("application_name", application_name))) @@ -329,11 +328,16 @@ function writestartupmessage( return nothing end +_write_messages_to_buffer(buf, debug, ::Tuple{}) = nothing +function _write_messages_to_buffer(buf, debug, msgs::Tuple) + _write_message_to_buffer(buf, debug, first(msgs)) + _write_messages_to_buffer(buf, debug, Base.tail(msgs)) + return nothing +end + function writemessages(socket, debug::Bool, msgs::Vararg{Tuple, N}) where {N} buf = IOBuffer() - for msg in msgs - _write_message_to_buffer(buf, debug, msg) - end + _write_messages_to_buffer(buf, debug, msgs) write(socket, take!(buf)) flush(socket) return @@ -829,7 +833,7 @@ function _startup!(socket, debug::Bool, user::String, dbname::String, @nospecial application_name_v = application_name::Union{String, Nothing} options_v = options::Union{String, Nothing} statement_timeout_v = statement_timeout::Union{Int, Nothing} - writestartupmessage(socket, debug, user, dbname, application_name_v, options_v, statement_timeout_v) + writestartupmessage(socket, debug, user, dbname, application_name_v, something(options_v, "")) # read initial response mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) if mt == UInt8('E') diff --git a/src/api/types.jl b/src/api/types.jl index 8341800..140cf97 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -407,6 +407,26 @@ function pg_parse_timestamp(s::AbstractString, ::Type{Durations.Timestamp{P}}) w return Durations.Timestamp{P}(Dates.UTInstant(P(Int64(ticks)))) end +# timestamp(tz) column into a Date field: the UTC date, as for DateTime fields +pg_parse_date_any(s::AbstractString)::Date = occursin(':', s) ? Date(pg_parse_datetime_any(s)) : pg_parse_date(s) + +# Integer fields also read integral numeric text such as "12.00" (numeric +# columns, or SUM over one). Out-of-range values throw InexactError, as when +# converting a decoded column value; other text throws the parser's error. +function pg_parse_integer(::Type{T}, s::AbstractString) where {T <: Integer} + x = Parsers.tryparse(T, s) + x === nothing || return x + wide = Parsers.tryparse(Int128, s) + if wide === nothing + dot = findfirst(==('.'), s) + if dot !== nothing && all(==('0'), SubString(s, dot + 1)) + wide = Parsers.tryparse(Int128, SubString(s, 1, dot - 1)) + end + end + wide === nothing && return Parsers.parse(T, s) + return convert(T, wide) +end + # timestamptz column into a DateTime field: sniff a trailing offset/Z function pg_parse_datetime_any(s::AbstractString)::DateTime isempty(s) && throw(ArgumentError("invalid postgres timestamp")) @@ -970,9 +990,35 @@ end const WireFieldScalar = Union{Durations.Timestamp, DataDecimals.AbstractDecimal, DateTime} _wire_field(::Type{T}) where {T} = T <: Union{Missing, WireFieldScalar} _wire_field(::Type{<:AbstractVector{T}}) where {T} = T <: WireFieldScalar -_field_source(::Type{T}, v::FieldValue) where {T} = _wire_field(T) ? _field_text(v.text) : parse_value(v.oid, v.text, v.registry) -StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = StructUtils.make(st, T, _field_source(T, v)) -StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue, tags) where {T} = StructUtils.make(st, T, _field_source(T, v), tags) + +_field_source(::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = + _wire_field(T) ? _field_text(v.text) : parse_value(v.oid, v.text, v.registry) + +# With the default style, field types that the lifts below (and StructUtils' +# own String, Symbol and Enum lifts) parse from text decode from the column +# text. The source stays a String, so typed reads compile to static calls +# under `--trim`; OID decoding returns `Any`. Custom styles keep the OID decoder +# so their lifts still receive decoded values, and other field types keep it too. +const TextField = Union{String, Symbol, Bool, Char, Int8, Int16, Int32, Int64, Cuint, Float32, Float64, + Date, Time, DateTime, UUID, Vector{UInt8}, JSONType, Vector{String}, Vector{Int16}, Vector{Int32}, + Vector{Int64}, Vector{Float32}, Vector{Float64}, Vector{Bool}, Vector{Date}, Vector{Time}, + Vector{DateTime}, Vector{UUID}, Vector{Char}} +_text_field(::Type{S}) where {S} = S !== Union{} && (S <: Union{TextField, Enum} || S === IntervalType || _wire_field(S)) + +function _field_source(::PostgresStyle, ::Type{T}, v::FieldValue) where {T} + S = Base.nonnothingtype(Base.nonmissingtype(T)) + _text_field(S) || return parse_value(v.oid, v.text, v.registry) + info = get(v.registry, v.oid, nothing) + if !_wire_field(S) && info !== nothing && info.parser !== nothing && info.julia_type <: S + # a parser registered for this column still applies when it produces + # the field's type (timestamps and decimals always parse the text) + x = info.parser(_field_text(v.text), v.registry) + x isa S && return x + end + return _field_text(v.text) +end +StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = StructUtils.make(st, T, _field_source(st, T, v)) +StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue, tags) where {T} = StructUtils.make(st, T, _field_source(st, T, v), tags) @static if isdefined(StructUtils, :InterpClosure) && isdefined(StructUtils, :HotStructClosure) @inline function applycast(f::Union{StructUtils.InterpClosure, StructUtils.HotStructClosure}, name, typeId, val, registry::Dict{Int, TypeInfo}) @@ -1000,16 +1046,16 @@ end return end -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int8}, s::String) = Parsers.parse(Int8, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int8}, s::String) = pg_parse_integer(Int8, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Bool}, s::String) = parse_boolean(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Char}, s::String) = pg_parse_char(s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int16}, s::String) = Parsers.parse(Int16, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int32}, s::String) = Parsers.parse(Int32, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int64}, s::String) = Parsers.parse(Int64, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Cuint}, s::String) = Parsers.parse(Cuint, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int16}, s::String) = pg_parse_integer(Int16, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int32}, s::String) = pg_parse_integer(Int32, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int64}, s::String) = pg_parse_integer(Int64, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Cuint}, s::String) = pg_parse_integer(Cuint, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Float32}, s::String) = Parsers.parse(Float32, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Float64}, s::String) = Parsers.parse(Float64, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Date}, s::String) = pg_parse_date(s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Date}, s::String) = pg_parse_date_any(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Time}, s::String) = pg_parse_time(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{DateTime}, s::String) = pg_parse_datetime_any(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{UUID}, s::String) = UUID(s), nothing diff --git a/src/array_parsing.jl b/src/array_parsing.jl index 7f27300..80390fa 100644 --- a/src/array_parsing.jl +++ b/src/array_parsing.jl @@ -4,7 +4,7 @@ using Parsers, Dates, UUIDs import Durations, DataDecimals import ..parse_decimal import ..pg_parse_timestamp -import ..pg_parse_date, ..pg_parse_time, ..pg_parse_datetime_any, ..pg_parse_char +import ..pg_parse_date, ..pg_parse_date_any, ..pg_parse_time, ..pg_parse_datetime_any, ..pg_parse_char, ..pg_parse_integer const BRACKET_OPEN = UInt8('[') const BRACKET_CLOSE = UInt8(']') @@ -88,12 +88,12 @@ function parse_scalar(token::String, inner_type::Type{T}, quoted::Bool) where {T !quoted && token == NULL_STR && return missing inner_type === String && return token inner_type === Bool && return parse_bool_token(token) - inner_type === Int16 && return Parsers.parse(Int16, token) - inner_type === Int32 && return Parsers.parse(Int32, token) - inner_type === Int64 && return Parsers.parse(Int64, token) + inner_type === Int16 && return pg_parse_integer(Int16, token) + inner_type === Int32 && return pg_parse_integer(Int32, token) + inner_type === Int64 && return pg_parse_integer(Int64, token) inner_type === Float32 && return Parsers.parse(Float32, token) inner_type === Float64 && return Parsers.parse(Float64, token) - inner_type === Date && return pg_parse_date(token) + inner_type === Date && return pg_parse_date_any(token) inner_type === Time && return pg_parse_time(token) inner_type <: Durations.Timestamp && return pg_parse_timestamp(token, inner_type) inner_type === DateTime && return pg_parse_datetime_any(token) diff --git a/src/execute.jl b/src/execute.jl index 6668b71..8f6761a 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -261,6 +261,19 @@ Base.close(cursor::Cursor) = DBInterface.close!(cursor) _param(x::AbstractString)::String = String(x) _param(x)::String = string(x) +# Same text as `string(x)`, whose Dates formatting is not trim-safe. +function _padded(n::Integer, width::Int) + digits = string(abs(n)) + for _ in ncodeunits(digits)+1:width + digits = string('0', digits) # lpad is not trim-safe + end + return n < 0 ? string('-', digits) : digits +end +_param(x::Date)::String = string(_padded(year(x), 4), '-', _padded(month(x), 2), '-', _padded(day(x), 2)) +function _param(x::DateTime)::String + text = string(_param(Date(x)), 'T', _padded(hour(x), 2), ':', _padded(minute(x), 2), ':', _padded(second(x), 2)) + return iszero(millisecond(x)) ? text : string(text, '.', _padded(millisecond(x), 3)) +end function _param(x::Durations.Timestamp)::String # PostgreSQL would round finer fractions. Require an exact conversion. timestamp = x isa Durations.Timestamp{Dates.Nanosecond} ? convert(API.PGTimestamp, x) : x @@ -332,6 +345,12 @@ function build_params(params, nparams::Int, sql::AbstractString) return dest end +function build_unchecked_params(params::Tuple) + dest = Vector{Union{String, Missing}}(undef, length(params)) + bind_tuple_params!(dest, params) + return dest +end + function build_unchecked_params(params) dest = Union{String, Missing}[] params === nothing && return dest diff --git a/test/result_decoding.jl b/test/result_decoding.jl index f6022f5..bddf8d0 100644 --- a/test/result_decoding.jl +++ b/test/result_decoding.jl @@ -18,6 +18,19 @@ function decoding_values(row) end struct DecodingStyle <: Postgres.AbstractPostgresStyle end + +@enum DecodingColor decoding_red decoding_green + +struct StringFields + json::String + jsonb::String + int::String + id::String + at::Union{Nothing, String} + amount::Union{Missing, String} + tags::String + note::Union{Nothing, String} +end StructUtils.lift(::DecodingStyle, ::Type{Int64}, value::Int32) = Int64(value) + 1, nothing function test_result_decoding() @@ -82,6 +95,54 @@ function test_result_decoding() end end + @testset "String fields take the column text" begin + texts = ["{\"a\": [1, 2]}", "{\"a\": [1, 2]}", "42", "c8b1cf79-de6a-54ab-a142-682c06a0de6a", + "2024-01-02 03:04:05.123+02:30", "12.50", "{a,b}", nothing] + row = decoding_row(texts, [114, 3802, 23, 2950, 1184, 1700, 1009, 25]; names=collect(fieldnames(StringFields))) + @test StructUtils.make(StringFields, row, Postgres.PostgresStyle()) == StringFields(texts[1:7]..., nothing) + target = NamedTuple{(:json, :jsonb), Tuple{String, JSONType}} + typed = StructUtils.make(target, decoding_row(texts[1:2], [114, 3802]; names=[:json, :jsonb]), Postgres.PostgresStyle()) + @test typed.json == texts[1] + @test JSON.parse(typed.jsonb)["a"] == [1, 2] + # untyped results keep the OID decoders + values = decoding_values(row) + @test values[2] isa JSONType && values[3] === Int32(42) && values[7] == ["a", "b"] + # a custom parser registered for a String column still applies + row = decoding_row(["abc"], [999_999]; names=[:value]) + Postgres.API.register_type!(row.type_registry, 999_999, String; parser=(text::String, registry) -> uppercase(text)) + @test StructUtils.make(NamedTuple{(:value,), Tuple{String}}, row, Postgres.PostgresStyle()).value == "ABC" + end + + @testset "Typed fields decode the column text" begin + # values the OID decoders converted exactly before still convert + texts = ["12.00", "7.0", "2024-01-02 23:30:00-05", "{12.00,3}", "5", "decoding_red"] + names = [:total, :count, :day, :amounts, :maybe, :color] + row = decoding_row(texts, [1700, 1700, 1184, 1231, 20, 25]; names=names) + target = NamedTuple{Tuple(names), Tuple{Int64, Int32, Date, Vector{Int64}, Union{Nothing, Int64}, DecodingColor}} + typed = StructUtils.make(target, row, Postgres.PostgresStyle()) + @test typed == (total=12, count=Int32(7), day=Date(2024, 1, 3), amounts=[12, 3], maybe=5, color=decoding_red) + @test_throws ArgumentError StructUtils.make(NamedTuple{(:total,), Tuple{Int64}}, + decoding_row(["12.50"], [1700]; names=[:total]), Postgres.PostgresStyle()) + # a registered decoder applies when it produces the field's type; + # otherwise the field decodes the text by its declared type + row = decoding_row(["decoding_red", "decoding_red"], [999_998, 999_998]; names=[:symbol, :color]) + Postgres.API.register_type!(row.type_registry, 999_998, Symbol; parser=(text::String, registry) -> Symbol(text)) + typed = StructUtils.make(NamedTuple{(:symbol, :color), Tuple{Symbol, DecodingColor}}, row, Postgres.PostgresStyle()) + @test typed == (symbol=:decoding_red, color=decoding_red) + end + + @testset "Date and DateTime parameters" begin + for x in (Date(2024, 1, 2), Date(12345, 1, 2), Date(-5, 1, 2), DateTime(2024, 1, 2, 3, 4, 5), + DateTime(2024, 1, 2, 3, 4, 5, 7), DateTime(2024, 1, 2, 3, 4, 5, 120), DateTime(-44, 3, 15, 12)) + @test Postgres._param(x) == string(x) + end + end + + @testset "Protocol strings reject NUL bytes" begin + @test_throws Postgres.PostgresInterfaceError Postgres.API.writepart(IOBuffer(), "a\0b") + @test_throws Postgres.PostgresInterfaceError Postgres.API.writepart(IOBuffer(), ("key", "a\0b")) + end + @testset "Decoded values own retained text" begin row = decoding_row(["hello α🙂", "{\"value\":\"saved α\"}", "custom β", "{first,second}", raw"\x0001ff"], [25, 3802, 999_999, 1009, 17]) From b2cb7a8fe9a398c3562bd5a1fa1fb3262457bc02 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 5 Oct 2026 16:22:17 -0600 Subject: [PATCH 2/8] Restore the juliac --trim compile test The 1.0 release removed test/trim_compile_tests.jl, so later changes to typed decoding regressed `--trim` builds without a failing test. The restored test compiles test/postgres_trim_queries.jl with JuliaC, requires zero verifier errors, and runs the executable against the test database (trust auth). The workload covers typed reads of nullable strings, dates and timestamps, numerics into Float64 and Int fields, enums, arrays, and jsonb into a String field, plus prepared statements, transactions and parameters of each bound type. JuliaC needs Julia 1.12, so it lives in test/trim/Project.toml and the test builds that environment only on released 1.12+ (64-bit, non-Windows), as JSON.jl does. Released Julia's Base64 pipes and Dates.CompoundPeriod constructor are not trim-safe; SASLAuth's SCRAM exchange and interval decoding reach them. Errors raised inside those stdlib files are reported and not counted, and the test compiles with `--trim=unsafe-warn` so the executable still builds. Co-Authored-By: Claude Opus 5.5 --- Project.toml | 4 +- test/postgres_trim_queries.jl | 151 ++++++++++++++++++++++++++++++++++ test/runtests.jl | 2 + test/trim/Project.toml | 4 + test/trim_compile_tests.jl | 109 ++++++++++++++++++++++++ 5 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 test/postgres_trim_queries.jl create mode 100644 test/trim/Project.toml create mode 100644 test/trim_compile_tests.jl diff --git a/Project.toml b/Project.toml index b0b5016..8f06212 100644 --- a/Project.toml +++ b/Project.toml @@ -31,6 +31,7 @@ Harbor = "1.1" JSON = "1" MD5 = "0.2" Parsers = "2.7, 3" +Pkg = "1.10" Random = "1.10" # Reseau 1.5 adds non-blocking tryread! behind isvalid. Reseau = "1.5" @@ -47,8 +48,9 @@ julia = "1.10" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Harbor = "af79dbb9-1a80-47ad-8928-192a4af69376" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Distributed", "Harbor", "Sockets", "Test"] +test = ["Aqua", "Distributed", "Harbor", "Pkg", "Sockets", "Test"] diff --git a/test/postgres_trim_queries.jl b/test/postgres_trim_queries.jl new file mode 100644 index 0000000..4b6dd11 --- /dev/null +++ b/test/postgres_trim_queries.jl @@ -0,0 +1,151 @@ +using Dates +using DBInterface +using Postgres +using StructUtils +using UUIDs + +@enum TrimStatus trim_active trim_paused + +struct TrimId + profile_id::Int32 +end + +StructUtils.@tags struct TrimProfile + profileId::Int32 &(postgres=(name=:profile_id,),) + displayName::String &(postgres=(name=:display_name,),) + nickname::Union{Nothing, String} + createdAt::DateTime &(postgres=(name=:created_at,),) + deletedAt::Union{Nothing, DateTime} &(postgres=(name=:deleted_at,),) + birthday::Union{Nothing, Date} + active::Bool + score::Union{Missing, Int32} + balance::Float64 + status::TrimStatus + uid::UUID + flags::Vector{Int32} + tags::Vector{String} + settings::String +end + +struct TrimTotals + total::Int64 + rows::Union{Nothing, Int64} +end + +const TRIM_PROFILE_SELECT = """ + SELECT profile_id, display_name, nickname, created_at, deleted_at, birthday, active, + score, balance, status, uid, flags, tags, settings + FROM trim_compile_profiles + """ + +function _postgres_trim_connect() + port = Base.parse(Int, get(ENV, "POSTGRES_TRIM_PORT", "5432")) + return DBInterface.connect( + Postgres.Connection, + get(ENV, "POSTGRES_TRIM_HOST", "127.0.0.1"), + get(ENV, "POSTGRES_TRIM_USER", "postgres"), + get(ENV, "POSTGRES_TRIM_PASSWORD", "postgres"); + dbname=get(ENV, "POSTGRES_TRIM_DBNAME", "postgres"), + port=port, + sslmode="disable", + connect_timeout=5, + application_name="postgres_trim", + options="-c search_path=public", + statement_timeout=30_000, + statement_cache_maxsize=4, + ) +end + +function _check(cond::Bool, msg::String)::Nothing + cond || error(msg) + return nothing +end + +function _check_profile(p::TrimProfile, id::Int32, name::String)::Nothing + _check(p.profileId == id, "unexpected profile id") + _check(p.displayName == name, "unexpected profile name") + _check(p.active, "expected active profile") + _check(!isempty(p.flags) && !isempty(p.tags), "expected non-empty arrays") + _check(startswith(p.settings, "{"), "expected jsonb text in a String field") + return nothing +end + +function run_postgres_trim_queries()::Nothing + conn = _postgres_trim_connect() + try + DBInterface.execute(conn, "DROP TYPE IF EXISTS trim_status") + DBInterface.execute(conn, "CREATE TYPE trim_status AS ENUM ('trim_active', 'trim_paused')") + DBInterface.execute(conn, """ + CREATE TEMP TABLE trim_compile_profiles ( + profile_id integer PRIMARY KEY, + display_name text NOT NULL, + nickname text, + created_at timestamp NOT NULL, + deleted_at timestamptz, + birthday date, + active boolean NOT NULL, + score integer, + balance numeric(10, 2) NOT NULL, + status trim_status NOT NULL, + uid uuid NOT NULL, + flags integer[] NOT NULL, + tags text[] NOT NULL, + settings jsonb NOT NULL + ) + """) + insert_sql = raw""" + INSERT INTO trim_compile_profiles VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9::numeric, $10::trim_status, $11, + $12::integer[], $13::text[], $14::jsonb + ) + RETURNING profile_id + """ + first_id = DBInterface.execute(conn, insert_sql, ( + Int32(1), "Ada", "ada", DateTime(2024, 1, 2, 3, 4, 5), missing, Date(1815, 12, 10), true, + Int32(99), "12.50", "trim_active", UUID("12345678-1234-5678-1234-567812345678"), + "{1,2,3}", "{math,poetry}", "{\"theme\": \"dark\"}", + ), TrimId) + _check(first_id.profile_id == 1, "unexpected inserted id") + DBInterface.transaction(conn) do + DBInterface.execute(conn, insert_sql, ( + Int32(2), "Grace", missing, DateTime(2024, 1, 3, 4, 5, 6), DateTime(2024, 2, 1), missing, true, + missing, "0.00", "trim_paused", UUID("87654321-4321-8765-4321-876543218765"), + "{4,5}", "{navy}", "{}", + )) + end + + stmt = DBInterface.prepare(conn, TRIM_PROFILE_SELECT * raw"WHERE profile_id = $1") + try + profile = DBInterface.execute(stmt, (Int32(1),), TrimProfile) + _check_profile(profile, Int32(1), "Ada") + _check(profile.nickname == "ada" && profile.deletedAt === nothing, "unexpected nullable fields") + _check(profile.balance == 12.5 && profile.status == trim_active, "unexpected numeric or enum field") + finally + DBInterface.close!(stmt) + end + + profiles = DBInterface.execute(conn, TRIM_PROFILE_SELECT * "ORDER BY profile_id", (), Vector{TrimProfile}) + _check(length(profiles) == 2, "expected two profiles") + _check_profile(profiles[2], Int32(2), "Grace") + _check(profiles[2].score === missing && profiles[2].deletedAt == DateTime(2024, 2, 1), "unexpected nullable fields") + + totals = DBInterface.execute(conn, + "SELECT sum(balance)::numeric(10, 2) AS total, count(*) AS rows FROM trim_compile_profiles WHERE balance = 0", + (), TrimTotals) + _check(totals.total == 0 && totals.rows == 1, "unexpected aggregate fields") + + result = DBInterface.execute(conn, "UPDATE trim_compile_profiles SET score = coalesce(score, 0) + 1") + _check(Postgres.rows_affected(result) == 2, "unexpected rows affected") + _check(occursin("UPDATE", Postgres.command_tag(result)), "unexpected command tag") + finally + DBInterface.close!(conn) + end + return nothing +end + +function @main(args::Vector{String})::Cint + run_postgres_trim_queries() + return 0 +end + +Base.Experimental.entrypoint(main, (Vector{String},)) diff --git a/test/runtests.jl b/test/runtests.jl index 71a4f5a..2581d74 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -500,6 +500,7 @@ include("isvalid_fake_server.jl") include("execute_fake_server.jl") include("executemany.jl") include("notification_deadlines.jl") +include("trim_compile_tests.jl") @testset "Postgres" begin test_timestamps() @@ -1050,6 +1051,7 @@ include("notification_deadlines.jl") conn = wait_for_connection(cfg) try test_executemany(conn) + run_postgres_trim_compile_tests(cfg) @testset "Auth" begin if occursin("trust", DEFAULT_AUTH) || occursin("trust", DEFAULT_INITDB_ARGS) conn_trust = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, nothing; dbname=cfg.dbname, port=cfg.port) diff --git a/test/trim/Project.toml b/test/trim/Project.toml new file mode 100644 index 0000000..fdf4b1d --- /dev/null +++ b/test/trim/Project.toml @@ -0,0 +1,4 @@ +[deps] +DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" +JuliaC = "acedd4c2-ced6-4a15-accc-2607eb759ba2" +StructUtils = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" diff --git a/test/trim_compile_tests.jl b/test/trim_compile_tests.jl new file mode 100644 index 0000000..3eb790f --- /dev/null +++ b/test/trim_compile_tests.jl @@ -0,0 +1,109 @@ +# Compiles test/postgres_trim_queries.jl with juliac, requires a clean verifier +# report apart from known stdlib gaps, and runs the executable against the test +# database. `--trim=unsafe-warn` reports the same verifier errors as `safe` but +# still builds the executable, so the stdlib gaps don't prevent the run. +import Pkg + +const _TRIM_SUPPORTED = VERSION >= v"1.12.0-rc1" && isempty(VERSION.prerelease) +const _JULIAC_ENTRYPOINT_EXPR = "using JuliaC; if isdefined(JuliaC, :main); JuliaC.main(ARGS); else JuliaC._main_cli(ARGS); end" +const _TRIM_COMPILE_TIMEOUT_S = parse(Float64, get(ENV, "POSTGRES_TRIM_COMPILE_TIMEOUT_S", "600")) +const _TRIM_RUN_TIMEOUT_S = parse(Float64, get(ENV, "POSTGRES_TRIM_EXE_TIMEOUT_S", "60")) + +function _prepare_trim_project(trim_project::String)::Nothing + mkpath(trim_project) + cp(joinpath(@__DIR__, "trim", "Project.toml"), joinpath(trim_project, "Project.toml")) + original_project = Base.active_project() + try + Pkg.activate(trim_project) + Pkg.develop(Pkg.PackageSpec(path = pkgdir(Postgres))) + Pkg.instantiate() + finally + original_project === nothing || Pkg.activate(original_project) + end + return nothing +end + +function _run_with_timeout(cmd::Cmd; timeout_s::Float64, label::String) + output_path = tempname() + exit_code, timed_out = -1, false + open(output_path, "w") do out + proc = run(pipeline(ignorestatus(cmd), stdout = out, stderr = out); wait = false) + started_at = time() + while Base.process_running(proc) + if time() - started_at >= timeout_s + timed_out = true + kill(proc) + break + end + sleep(0.1) + end + timed_out || wait(proc) + exit_code = something(proc.exitcode, -1) + end + output = read(output_path, String) + rm(output_path; force = true) + timed_out && error("trim $(label) timed out after $(timeout_s)s:\n$(output)") + return exit_code, output +end + +# Released Julia's Base64 pipes and Dates.CompoundPeriod constructor are not +# trim-safe (SASLAuth's SCRAM exchange and interval decoding reach them). An +# error whose stdlib frames, before the first package frame, pass through those +# files is reported but not counted. +function _stdlib_trim_gap(error_text::AbstractString)::Bool + for m in eachmatch(r"\n\s+@ (\S+) (\S+):\d+", error_text) + mod, path = m.captures + occursin(r"/Base64/src/|/Dates/src/periods\.jl", path) && return true + (mod == "Core" || mod == "Base" || startswith(mod, "Base.")) || return false + end + return false +end + +function _trim_verify_counts(output::String) + blocks = split(output, r"\nVerifier (?=error #\d+:)")[2:end] + gaps = count(_stdlib_trim_gap, blocks) + warnings = length(collect(eachmatch(r"Verifier warning #\d+:", output))) + return length(blocks) - gaps, gaps, warnings +end + +function run_postgres_trim_compile_tests(cfg)::Nothing + @testset "Trim compile" begin + if Sys.iswindows() || Sys.WORD_SIZE != 64 || !_TRIM_SUPPORTED + println("[trim] skipped: JuliaC trim compilation runs on 64-bit Linux/macOS with released Julia 1.12+") + @test true + return nothing + end + mktempdir() do tmpdir + trim_project = joinpath(tmpdir, "trim_project") + _prepare_trim_project(trim_project) + julia = joinpath(Sys.BINDIR, Base.julia_exename()) + script = joinpath(@__DIR__, "postgres_trim_queries.jl") + exe = joinpath(tmpdir, "postgres_trim_queries") + # --output-exe takes a bare name, written to the working directory + compile = Cmd(`$julia --startup-file=no --history-file=no --code-coverage=none --project=$trim_project -e $(_JULIAC_ENTRYPOINT_EXPR) -- --output-exe postgres_trim_queries --project=$trim_project --experimental --trim=unsafe-warn $script`; dir = tmpdir) + exit_code, output = _run_with_timeout(compile; timeout_s = _TRIM_COMPILE_TIMEOUT_S, label = "compile") + errors, stdlib_errors, warnings = _trim_verify_counts(output) + println("[trim] verifier errors=$(errors) stdlib_errors=$(stdlib_errors) warnings=$(warnings)") + (errors > 0 || warnings > 0 || exit_code != 0) && println(output) + @test errors == 0 + @test warnings == 0 + @test exit_code == 0 + (errors == 0 && exit_code == 0) || return nothing + # password auth would run SASLAuth's SCRAM exchange, one of the stdlib gaps + if !(occursin("trust", DEFAULT_AUTH) || occursin("trust", DEFAULT_INITDB_ARGS)) + println("[trim] executable run skipped: needs trust auth") + return nothing + end + env = copy(ENV) + env["POSTGRES_TRIM_HOST"] = cfg.host + env["POSTGRES_TRIM_PORT"] = string(cfg.port) + env["POSTGRES_TRIM_USER"] = cfg.user + env["POSTGRES_TRIM_PASSWORD"] = cfg.password + env["POSTGRES_TRIM_DBNAME"] = cfg.dbname + run_exit, run_output = _run_with_timeout(setenv(`$exe`, env); timeout_s = _TRIM_RUN_TIMEOUT_S, label = "run") + run_exit == 0 || println(run_output) + @test run_exit == 0 + end + end + return nothing +end From 46294c59180d4d5429b4f947111a7ff2cd85f086 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 5 Oct 2026 16:52:48 -0600 Subject: [PATCH 3/8] Bump version to 2.3.0 Co-Authored-By: Claude Opus 5.5 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 8f06212..e7fefc1 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Postgres" uuid = "8f23287e-300e-4f50-bc2b-9f1dfe95da84" -version = "2.2.2" +version = "2.3.0" [deps] ConcurrentUtilities = "f0e56b4a-5159-44fe-b623-3e5288b988bb" From 36feebc668cfe762c3f1d979dbe5e7aae63cfd26 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 5 Oct 2026 17:05:53 -0600 Subject: [PATCH 4/8] test: wait for krbuser backends to exit before dropping the role The Kerberos fixture's cleanup ran `DROP ROLE krbuser` right after closing the last krbuser connection. That connection created a temp table, and PostgreSQL drops a session's temp tables only as its backend exits, which can trail `close!`. On a fast runner the DROP ROLE won the race and failed with 2BP01 ("owner of table pg_temp_4.bulk_gss"). Poll pg_stat_activity until the role has no sessions (up to 10s) before dropping it. Co-Authored-By: Claude Opus 5.5 --- test/gssapi.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/gssapi.jl b/test/gssapi.jl index f0317f7..85e7453 100644 --- a/test/gssapi.jl +++ b/test/gssapi.jl @@ -668,6 +668,12 @@ function test_kerberos_integration() err = connect_err(() -> DBInterface.connect(Postgres.Connection, cfg.host, KRB_USER, nothing; dbname=cfg.dbname, port=cfg.port, gssencmode="prefer", krbsrvname="nosuchservice")) @test err isa GSSAPI.GSSError finally + # a closed session's temp tables are dropped as its backend exits, + # which can trail close!; DROP ROLE fails while the role owns them + for _ in 1:100 + only(DBInterface.execute(admin, "SELECT count(*) AS n FROM pg_stat_activity WHERE usename = '$KRB_USER'")).n == 0 && break + sleep(0.1) + end DBInterface.execute(admin, "DROP ROLE IF EXISTS $KRB_USER") DBInterface.close!(admin) end From 6c608c1f658830913f14bad2d49c1a89f74c8fe4 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 26 Oct 2026 17:43:53 -0600 Subject: [PATCH 5/8] Bump version to 2.3.1 Co-Authored-By: Claude Opus 5.5 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index e7fefc1..b59c721 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Postgres" uuid = "8f23287e-300e-4f50-bc2b-9f1dfe95da84" -version = "2.3.0" +version = "2.3.1" [deps] ConcurrentUtilities = "f0e56b4a-5159-44fe-b623-3e5288b988bb" From aedee490f7add8902450617647c274bbd3473425 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Tue, 27 Oct 2026 12:16:40 -0600 Subject: [PATCH 6/8] Preserve typed read conversions and verify compiler diagnostics Use the source floating precision for numeric conversions, preserve union selection, and extract UTC times from timestamps. Cover scalar, nullable, and array reads against PostgreSQL and in the compiled workload. Recognize current and legacy trim diagnostics, check their reported totals, and capture the complete compiler output through a pipe. Keep the existing stdlib exemptions and verify the executable against a live database. --- src/api/types.jl | 30 ++++++-- src/array_parsing.jl | 4 +- test/postgres_trim_queries.jl | 37 ++++++++- test/result_decoding.jl | 136 ++++++++++++++++++++++++++++++++++ test/runtests.jl | 2 + test/trim_compile_tests.jl | 77 ++++++++++++++----- 6 files changed, 258 insertions(+), 28 deletions(-) diff --git a/src/api/types.jl b/src/api/types.jl index 75eacb7..f11885f 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -410,6 +410,12 @@ end # timestamp(tz) column into a Date field: the UTC date, as for DateTime fields pg_parse_date_any(s::AbstractString)::Date = occursin(':', s) ? Date(pg_parse_datetime_any(s)) : pg_parse_date(s) +# Timestamp columns contribute their UTC time of day without losing microseconds. +function pg_parse_time_any(s::AbstractString)::Time + _check_temporal_special(s, "timestamp") + return occursin(' ', s) ? convert(Time, pg_parse_timestamp(s)) : pg_parse_time(s) +end + # Integer fields also read integral numeric text such as "12.00" (numeric # columns, or SUM over one). Out-of-range values throw InexactError, as when # converting a decoded column value; other text throws the parser's error. @@ -996,14 +1002,15 @@ _field_source(::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = # With the default style, field types that the lifts below (and StructUtils' # own String, Symbol and Enum lifts) parse from text decode from the column -# text. The source stays a String, so typed reads compile to static calls -# under `--trim`; OID decoding returns `Any`. Custom styles keep the OID decoder -# so their lifts still receive decoded values, and other field types keep it too. +# text. Floating columns retain their precision for numeric targets. These +# sources have inferred types under `--trim`; OID decoding returns `Any`. +# Custom styles keep the OID decoder so their lifts still receive decoded values, +# and other field types keep it too. const TextField = Union{String, Symbol, Bool, Char, Int8, Int16, Int32, Int64, Cuint, Float32, Float64, Date, Time, DateTime, UUID, Vector{UInt8}, JSONType, Vector{String}, Vector{Int16}, Vector{Int32}, Vector{Int64}, Vector{Float32}, Vector{Float64}, Vector{Bool}, Vector{Date}, Vector{Time}, Vector{DateTime}, Vector{UUID}, Vector{Char}} -_text_field(::Type{S}) where {S} = S !== Union{} && (S <: Union{TextField, Enum} || S === IntervalType || _wire_field(S)) +_text_field(::Type{S}) where {S} = S === IntervalType || (S !== Union{} && !(S isa Union) && (S <: Union{TextField, Enum} || _wire_field(S))) function _field_source(::PostgresStyle, ::Type{T}, v::FieldValue) where {T} S = Base.nonnothingtype(Base.nonmissingtype(T)) @@ -1015,7 +1022,18 @@ function _field_source(::PostgresStyle, ::Type{T}, v::FieldValue) where {T} x = info.parser(_field_text(v.text), v.registry) x isa S && return x end - return _field_text(v.text) + text = _field_text(v.text) + # Conversions must use the column's floating precision, including exponent + # output. Parsing float4 directly as Float64 would silently change its value. + if S <: Union{Signed, Unsigned, AbstractFloat} + v.oid == 700 && return convert(S, Parsers.parse(Float32, text)) + v.oid == 701 && return convert(S, Parsers.parse(Float64, text)) + elseif S <: Union{Vector{<:Signed}, Vector{<:AbstractFloat}} + # These concrete targets require flat arrays with no NULL elements. + v.oid == 1021 && return convert(S, parse_array(text, Float32)::Vector{Float32}) + v.oid == 1022 && return convert(S, parse_array(text, Float64)::Vector{Float64}) + end + return text end StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = StructUtils.make(st, T, _field_source(st, T, v)) StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue, tags) where {T} = StructUtils.make(st, T, _field_source(st, T, v), tags) @@ -1056,7 +1074,7 @@ StructUtils.lift(::AbstractPostgresStyle, ::Type{Cuint}, s::String) = pg_parse_i StructUtils.lift(::AbstractPostgresStyle, ::Type{Float32}, s::String) = Parsers.parse(Float32, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Float64}, s::String) = Parsers.parse(Float64, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Date}, s::String) = pg_parse_date_any(s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Time}, s::String) = pg_parse_time(s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Time}, s::String) = pg_parse_time_any(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{DateTime}, s::String) = pg_parse_datetime_any(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{UUID}, s::String) = UUID(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{IntervalType}, s::String) = parse_interval(s), nothing diff --git a/src/array_parsing.jl b/src/array_parsing.jl index 80390fa..919356f 100644 --- a/src/array_parsing.jl +++ b/src/array_parsing.jl @@ -4,7 +4,7 @@ using Parsers, Dates, UUIDs import Durations, DataDecimals import ..parse_decimal import ..pg_parse_timestamp -import ..pg_parse_date, ..pg_parse_date_any, ..pg_parse_time, ..pg_parse_datetime_any, ..pg_parse_char, ..pg_parse_integer +import ..pg_parse_date, ..pg_parse_date_any, ..pg_parse_time_any, ..pg_parse_datetime_any, ..pg_parse_char, ..pg_parse_integer const BRACKET_OPEN = UInt8('[') const BRACKET_CLOSE = UInt8(']') @@ -94,7 +94,7 @@ function parse_scalar(token::String, inner_type::Type{T}, quoted::Bool) where {T inner_type === Float32 && return Parsers.parse(Float32, token) inner_type === Float64 && return Parsers.parse(Float64, token) inner_type === Date && return pg_parse_date_any(token) - inner_type === Time && return pg_parse_time(token) + inner_type === Time && return pg_parse_time_any(token) inner_type <: Durations.Timestamp && return pg_parse_timestamp(token, inner_type) inner_type === DateTime && return pg_parse_datetime_any(token) inner_type === UUID && return UUID(token) diff --git a/test/postgres_trim_queries.jl b/test/postgres_trim_queries.jl index 4b6dd11..aa3faaf 100644 --- a/test/postgres_trim_queries.jl +++ b/test/postgres_trim_queries.jl @@ -32,6 +32,21 @@ struct TrimTotals rows::Union{Nothing, Int64} end +struct TrimCompatibility + float4_int::Int64 + float8_int::Union{Nothing, Int64} + absent_int::Union{Missing, Int64} + widened::Float64 + integers::Vector{Int64} + widened_array::Vector{Float64} + clock::Time + maybe_clock::Union{Nothing, Time} + absent_clock::Union{Missing, Time} + clocks::Vector{Time} + json_null::Union{Nothing, String} + sql_null::Union{Nothing, String} +end + const TRIM_PROFILE_SELECT = """ SELECT profile_id, display_name, nickname, created_at, deleted_at, birthday, active, score, balance, status, uid, flags, tags, settings @@ -50,7 +65,7 @@ function _postgres_trim_connect() sslmode="disable", connect_timeout=5, application_name="postgres_trim", - options="-c search_path=public", + options="-c search_path=public -c timezone=UTC", statement_timeout=30_000, statement_cache_maxsize=4, ) @@ -73,6 +88,26 @@ end function run_postgres_trim_queries()::Nothing conn = _postgres_trim_connect() try + DBInterface.execute(conn, "SET TimeZone = 'Asia/Kolkata'") + compat = DBInterface.execute(conn, """ + SELECT 1e15::float4 AS float4_int, 1e15::float8 AS float8_int, + NULL::float8 AS absent_int, 0.1::float4 AS widened, + ARRAY[1e15::float4, -1e15::float4] AS integers, + ARRAY[0.1::float4] AS widened_array, + '1969-12-31 23:59:59.999999'::timestamp AS clock, + '1800-01-01 00:00:00.123456+00'::timestamptz AS maybe_clock, + NULL::timestamp AS absent_clock, + ARRAY['1970-01-01 05:21:09.999999+05:21:10'::timestamptz] AS clocks, + 'null'::jsonb AS json_null, NULL::jsonb AS sql_null + """, (), TrimCompatibility) + _check(compat.float4_int == 999999986991104 && compat.float8_int === Int64(1000000000000000), "unexpected exponent conversion") + _check(compat.absent_int === missing, "unexpected nullable integer") + _check(compat.widened === Float64(0.1f0) && compat.widened_array == [Float64(0.1f0)], "lost float4 precision") + _check(compat.integers == Int64[999999986991104, -999999986991104], "unexpected exponent array conversion") + _check(compat.clock === Time(23, 59, 59, 999, 999) && compat.clocks == [compat.clock], "lost timestamp time precision") + _check(compat.maybe_clock === Time(0, 0, 0, 123, 456) && compat.absent_clock === missing, "unexpected timestamp offset or NULL") + _check(compat.json_null === "null" && compat.sql_null === nothing, "confused JSON null with SQL NULL") + DBInterface.execute(conn, "RESET TimeZone") DBInterface.execute(conn, "DROP TYPE IF EXISTS trim_status") DBInterface.execute(conn, "CREATE TYPE trim_status AS ENUM ('trim_active', 'trim_paused')") DBInterface.execute(conn, """ diff --git a/test/result_decoding.jl b/test/result_decoding.jl index bddf8d0..10d261e 100644 --- a/test/result_decoding.jl +++ b/test/result_decoding.jl @@ -11,12 +11,68 @@ function decoding_row(values, oids; names=[Symbol("c", i) for i in eachindex(val return Postgres.API.DataRow(take!(io), names, oids, Dict(Postgres.API.DEFAULT_TYPE_REGISTRY)) end +function test_typed_read_compatibility(conn) + @testset "Typed read compatibility on PostgreSQL" begin + settings = ("SET LOCAL TimeZone = 'UTC'", "SET LOCAL TimeZone = 'Asia/Kolkata'", + "SET LOCAL TimeZone = 'America/New_York'", "SET LOCAL IntervalStyle = 'iso_8601'", + "SET LOCAL bytea_output = 'escape'", "SET LOCAL extra_float_digits = 0", + "SET LOCAL extra_float_digits = 3") + for setting in settings + DBInterface.transaction(conn) do + DBInterface.execute(conn, "SET LOCAL DateStyle = 'ISO, YMD'") + DBInterface.execute(conn, setting) + for pgtype in ("float4", "float8") + sql = "SELECT 1e15::$pgtype AS value" + decoded = only(DBInterface.execute(conn, sql)).value + for T in (Int64, Union{Nothing, Int64}, Union{Missing, Int64}) + @test DBInterface.execute(conn, sql, (), NamedTuple{(:value,), Tuple{T}}).value === Int64(decoded) + end + for value in ("3.5", "1e19", "'NaN'", "'Infinity'") + @test_throws InexactError DBInterface.execute(conn, "SELECT $value::$pgtype AS value", (), NamedTuple{(:value,), Tuple{Int64}}) + end + @test DBInterface.execute(conn, "SELECT ARRAY[1e15::$pgtype, -1e15::$pgtype] AS value", (), + NamedTuple{(:value,), Tuple{Vector{Int64}}}).value == [Int64(decoded), -Int64(decoded)] + end + widened = DBInterface.execute(conn, "SELECT 0.1::float4 AS value", (), NamedTuple{(:value,), Tuple{Float64}}).value + @test widened === Float64(0.1f0) + for (expr, T, expected) in (("42::int8", Union{Int64, String}, Int64(42)), + ("'42'::text", Union{Int64, String}, "42"), + ("1.5::float8", Union{Int64, Float64}, 1.5), + ("NULL::float8", Union{Missing, Int64, Float64}, missing), + ("'null'::json", Union{Nothing, String}, "null"), + ("NULL::jsonb", Union{Nothing, String}, nothing)) + @test DBInterface.execute(conn, "SELECT $expr AS value", (), NamedTuple{(:value,), Tuple{T}}).value === expected + end + # The old date forces named zones to emit historical second offsets. + for expr in ("'1800-01-01 00:00:00.123456'::timestamp", "'1800-01-01 00:00:00.123456+00'::timestamptz") + expected = Time(0, 0, 0, 123, 456) + for T in (Time, Union{Nothing, Time}, Union{Missing, Time}) + @test DBInterface.execute(conn, "SELECT $expr AS value", (), NamedTuple{(:value,), Tuple{T}}).value === expected + end + @test DBInterface.execute(conn, "SELECT ARRAY[$expr] AS value", (), NamedTuple{(:value,), Tuple{Vector{Time}}}).value == [expected] + @test isequal(DBInterface.execute(conn, "SELECT ARRAY[$expr, NULL] AS value", (), + NamedTuple{(:value,), Tuple{Vector{Union{Missing, Time}}}}).value, [expected, missing]) + end + for (expr, expected) in (("9223372036854775807.00::numeric", typemax(Int64)), + ("-9223372036854775808.00::numeric", typemin(Int64))) + @test DBInterface.execute(conn, "SELECT $expr AS value", (), NamedTuple{(:value,), Tuple{Int64}}).value === expected + end + @test_throws InexactError DBInterface.execute(conn, "SELECT 9223372036854775808::numeric AS value", (), NamedTuple{(:value,), Tuple{Int64}}) + @test_throws ArgumentError DBInterface.execute(conn, "SELECT 9007199254740993.5::numeric AS value", (), NamedTuple{(:value,), Tuple{Int64}}) + end + end + end +end + function decoding_values(row) values = Any[] StructUtils.applyeach(Postgres.PostgresStyle(), (name, value) -> push!(values, value), row) return values end +decoding_field(::Type{T}, text, oid) where {T} = StructUtils.make( + NamedTuple{(:value,), Tuple{T}}, decoding_row([text], [oid]; names=[:value]), Postgres.PostgresStyle()).value + struct DecodingStyle <: Postgres.AbstractPostgresStyle end @enum DecodingColor decoding_red decoding_green @@ -131,6 +187,86 @@ function test_result_decoding() @test typed == (symbol=:decoding_red, color=decoding_red) end + @testset "Floating columns retain their precision in typed conversions" begin + for (oid, array_oid, F) in ((700, 1021, Float32), (701, 1022, Float64)) + for text in ("1e+15", "-1e+15", "1e+18", "-1e+18", "9.223371e18", "-9.223372e18"), T in (Int64, Union{Nothing, Int64}, Union{Missing, Int64}) + @test decoding_field(T, text, oid) === Int64(parse(F, text)) + end + for text in ("3.5", "1e-2", "1e+19", "-1e+19", "NaN", "Infinity", "-Infinity") + @test_throws InexactError decoding_field(Int64, text, oid) + @test_throws InexactError decoding_field(Vector{Int64}, "{$text}", array_oid) + end + @test decoding_field(Int8, "1e+2", oid) === Int8(100) + @test_throws InexactError decoding_field(Int8, "1.28e+2", oid) + @test decoding_field(Cuint, "1e+2", oid) === Cuint(100) + @test_throws InexactError decoding_field(Cuint, "-1e+2", oid) + for T in (Vector{Int64}, Union{Nothing, Vector{Int64}}, Union{Missing, Vector{Int64}}) + @test decoding_field(T, "{1e+15,-1e+15}", array_oid) == Int64[parse(F, "1e+15"), parse(F, "-1e+15")] + end + end + @test decoding_field(Float64, "0.1", 700) === Float64(0.1f0) + @test decoding_field(Vector{Float64}, "{0.1}", 1021) == [Float64(0.1f0)] + @test decoding_field(Int64, "9223372036854775807.00", 1700) === typemax(Int64) + @test decoding_field(Int64, "-9223372036854775808.00", 1700) === typemin(Int64) + @test_throws InexactError decoding_field(Int64, "9223372036854775808.00", 1700) + @test_throws ArgumentError decoding_field(Int64, "9007199254740993.5", 1700) + # Exponent-form text is not a floating column and must not round through Float64. + @test_throws ArgumentError decoding_field(Int64, "9007199254740993e0", 25) + for (T, expected) in ((Union{Nothing, Int64}, nothing), (Union{Missing, Int64}, missing), + (Union{Nothing, Vector{Int64}}, nothing), (Union{Missing, Vector{Int64}}, missing)) + @test decoding_field(T, nothing, 701) === expected + end + row = decoding_row(["1e+15"], [701]; names=[:value]) + Postgres.API.register_type!(row.type_registry, 701, Int64; parser=(text::String, registry) -> Int64(42)) + @test StructUtils.make(NamedTuple{(:value,), Tuple{Int64}}, row, Postgres.PostgresStyle()).value === Int64(42) + end + + @testset "General unions retain the decoded column type" begin + for T in (Union{Int64, Float64}, Union{Nothing, Int64, Float64}, Union{Missing, Int64, Float64}) + @test decoding_field(T, "1.5", 701) === 1.5 + @test decoding_field(T, "42", 20) === Int64(42) + end + for T in (Union{Int64, String}, Union{Nothing, Int64, String}, Union{Missing, Int64, String}) + @test decoding_field(T, "42", 20) === Int64(42) + @test decoding_field(T, "42", 25) === "42" + end + @test decoding_field(Union{Nothing, Int64, String}, nothing, 20) === nothing + @test decoding_field(Union{Missing, Int64, Float64}, nothing, 701) === missing + @test decoding_field(Vector{Union{Int64, String}}, "{42}", 1016) == Int64[42] + @test isequal(decoding_field(Vector{Union{Missing, Int64, Float64}}, "{1.5,NULL}", 1022), [1.5, missing]) + # IntervalType is an intentional union with its own text lift. + @test decoding_field(Postgres.API.IntervalType, "1 year 2 mons", 1186) == Year(1) + Month(2) + end + + @testset "Timestamp columns into Time fields" begin + expected = Time(23, 59, 59, 999, 999) + for (oid, array_oid, text) in ((1114, 1115, "1969-12-31 23:59:59.999999"), + (1184, 1185, "1970-01-01 05:21:09.999999+05:21:10"), + (1184, 1185, "1969-12-31 18:38:49.999999-05:21:10")) + for T in (Time, Union{Nothing, Time}, Union{Missing, Time}) + @test decoding_field(T, text, oid) === expected + end + for T in (Vector{Time}, Union{Nothing, Vector{Time}}, Union{Missing, Vector{Time}}) + @test decoding_field(T, "{\"$text\"}", array_oid) == [expected] + end + end + @test decoding_field(Time, "03:04:05.123456789", 1083) === Time(3, 4, 5, 123, 456, 789) + @test decoding_field(Union{Nothing, Time}, nothing, 1114) === nothing + @test decoding_field(Union{Missing, Time}, nothing, 1184) === missing + @test decoding_field(Union{Nothing, Vector{Time}}, nothing, 1185) === nothing + @test_throws Postgres.PostgresInterfaceError decoding_field(Time, "infinity", 1114) + @test_throws Postgres.PostgresInterfaceError decoding_field(Time, "0001-01-01 00:00:00 BC", 1114) + end + + @testset "JSON null stays distinct from SQL NULL in text fields" begin + for oid in (114, 3802) + @test decoding_field(Union{Nothing, String}, "null", oid) === "null" + @test decoding_field(Union{Missing, String}, "null", oid) === "null" + @test decoding_field(Union{Nothing, String}, nothing, oid) === nothing + @test decoding_field(Union{Missing, String}, nothing, oid) === missing + end + end + @testset "Date and DateTime parameters" begin for x in (Date(2024, 1, 2), Date(12345, 1, 2), Date(-5, 1, 2), DateTime(2024, 1, 2, 3, 4, 5), DateTime(2024, 1, 2, 3, 4, 5, 7), DateTime(2024, 1, 2, 3, 4, 5, 120), DateTime(-44, 3, 15, 12)) diff --git a/test/runtests.jl b/test/runtests.jl index 254ed19..36aa166 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1042,6 +1042,7 @@ include("trim_compile_tests.jl") end test_result_decoding() + test_trim_verifier_diagnostics() test_gssapi() test_isvalid_fake_server() test_isvalid_fragmentation() @@ -1061,6 +1062,7 @@ include("trim_compile_tests.jl") conn = wait_for_connection(cfg) try test_executemany(conn) + test_typed_read_compatibility(conn) run_postgres_trim_compile_tests(cfg) @testset "Auth" begin if occursin("trust", DEFAULT_AUTH) || occursin("trust", DEFAULT_INITDB_ARGS) diff --git a/test/trim_compile_tests.jl b/test/trim_compile_tests.jl index 3eb790f..79285e1 100644 --- a/test/trim_compile_tests.jl +++ b/test/trim_compile_tests.jl @@ -1,7 +1,7 @@ # Compiles test/postgres_trim_queries.jl with juliac, requires a clean verifier # report apart from known stdlib gaps, and runs the executable against the test -# database. `--trim=unsafe-warn` reports the same verifier errors as `safe` but -# still builds the executable, so the stdlib gaps don't prevent the run. +# database. `--trim=unsafe-warn` builds despite reported errors. Executing this +# workload verifies its runtime paths, not strict `--trim=safe` support. import Pkg const _TRIM_SUPPORTED = VERSION >= v"1.12.0-rc1" && isempty(VERSION.prerelease) @@ -24,24 +24,34 @@ function _prepare_trim_project(trim_project::String)::Nothing end function _run_with_timeout(cmd::Cmd; timeout_s::Float64, label::String) - output_path = tempname() + # JuliaC and its compiler subprocess share these streams. A pipe keeps + # their writes ordered without competing regular-file offsets. + pipe = Pipe() + captured = IOBuffer() exit_code, timed_out = -1, false - open(output_path, "w") do out - proc = run(pipeline(ignorestatus(cmd), stdout = out, stderr = out); wait = false) - started_at = time() - while Base.process_running(proc) - if time() - started_at >= timeout_s - timed_out = true - kill(proc) - break - end - sleep(0.1) + proc = run(pipeline(ignorestatus(cmd), stdout = pipe, stderr = pipe); wait = false) + close(pipe.in) + reader = @async try + while !eof(pipe) + write(captured, readavailable(pipe)) + end + catch err + timed_out || rethrow() + end + started_at = time() + while Base.process_running(proc) + if time() - started_at >= timeout_s + timed_out = true + kill(proc) + break end - timed_out || wait(proc) - exit_code = something(proc.exitcode, -1) + sleep(0.1) end - output = read(output_path, String) - rm(output_path; force = true) + timed_out ? close(pipe) : wait(proc) + wait(reader) + close(pipe) + exit_code = something(proc.exitcode, -1) + output = String(take!(captured)) timed_out && error("trim $(label) timed out after $(timeout_s)s:\n$(output)") return exit_code, output end @@ -60,12 +70,41 @@ function _stdlib_trim_gap(error_text::AbstractString)::Bool end function _trim_verify_counts(output::String) - blocks = split(output, r"\nVerifier (?=error #\d+:)")[2:end] + blocks = split("\n" * output, r"\n(?:Verifier error|Error) #\d+:")[2:end] gaps = count(_stdlib_trim_gap, blocks) - warnings = length(collect(eachmatch(r"Verifier warning #\d+:", output))) + warnings = length(collect(eachmatch(r"(?m)^(?:Verifier warning|Warning) #\d+:", output))) + summary = match(r"Trim verify finished with (\d+) errors?, (\d+) warnings?", output) + if summary !== nothing + reported = parse(Int, summary[1]) + parse(Int, summary[2]) + length(blocks) + warnings == reported || error("unrecognized trim verifier diagnostics:\n$output") + end return length(blocks) - gaps, gaps, warnings end +function test_trim_verifier_diagnostics() + @testset "Trim verifier diagnostics" begin + package_frame = "\nStacktrace:\n [1] parse_value(...)\n @ Postgres.API /tmp/Postgres/src/api/types.jl:1\n" + stdlib_frame = "\nStacktrace:\n [1] write(...)\n @ Base64 /julia/share/julia/stdlib/v1.13/Base64/src/encode.jl:1\n" + for prefix in ("Verifier error", "Error") + output = "$prefix #1: unresolved call$package_frame\n$prefix #2: unresolved call$stdlib_frame\n$prefix #3: unresolved call$package_frame" + @test _trim_verify_counts(output) == (2, 1, 0) + end + for prefix in ("Verifier warning", "Warning") + @test _trim_verify_counts("$prefix #1: unresolved call") == (0, 0, 1) + end + @test _trim_verify_counts("Compiling...\n") == (0, 0, 0) + @test _trim_verify_counts("Error #1: unresolved call$package_frame\nTrim verify finished with 1 error, 0 warnings.") == (1, 0, 0) + @test_throws ErrorException _trim_verify_counts("Trim verify finished with 1 error, 0 warnings.") + + julia = joinpath(Sys.BINDIR, Base.julia_exename()) + child = `$julia --startup-file=no -e $("print(stderr, repeat(\"x\", 65536)); flush(stderr)")` + script = "print(stderr, \"start\\n\"); flush(stderr); run(pipeline($(repr(child)); stdout, stderr)); print(stderr, \"end\\n\"); flush(stderr)" + code, output = _run_with_timeout(`$julia --startup-file=no -e $script`; timeout_s=30.0, label="capture test") + @test code == 0 + @test output == "start\n" * repeat("x", 65536) * "end\n" + end +end + function run_postgres_trim_compile_tests(cfg)::Nothing @testset "Trim compile" begin if Sys.iswindows() || Sys.WORD_SIZE != 64 || !_TRIM_SUPPORTED From 1a2c703e07893481c630ccacb77b3b7c47f2e0aa Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Tue, 27 Oct 2026 12:23:41 -0600 Subject: [PATCH 7/8] Preserve unsigned numeric array conversions --- src/api/types.jl | 8 ++++---- test/postgres_trim_queries.jl | 3 +++ test/result_decoding.jl | 5 +++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/api/types.jl b/src/api/types.jl index f11885f..1bdab37 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -416,9 +416,9 @@ function pg_parse_time_any(s::AbstractString)::Time return occursin(' ', s) ? convert(Time, pg_parse_timestamp(s)) : pg_parse_time(s) end -# Integer fields also read integral numeric text such as "12.00" (numeric -# columns, or SUM over one). Out-of-range values throw InexactError, as when -# converting a decoded column value; other text throws the parser's error. +# Integer fields also read integral numeric text such as "12.00". Widening to +# Int128 detects values outside the target range and throws InexactError; +# text outside Int128's range or in other forms throws the target parser's error. function pg_parse_integer(::Type{T}, s::AbstractString) where {T <: Integer} x = Parsers.tryparse(T, s) x === nothing || return x @@ -1028,7 +1028,7 @@ function _field_source(::PostgresStyle, ::Type{T}, v::FieldValue) where {T} if S <: Union{Signed, Unsigned, AbstractFloat} v.oid == 700 && return convert(S, Parsers.parse(Float32, text)) v.oid == 701 && return convert(S, Parsers.parse(Float64, text)) - elseif S <: Union{Vector{<:Signed}, Vector{<:AbstractFloat}} + elseif S <: Union{Vector{<:Signed}, Vector{<:Unsigned}, Vector{<:AbstractFloat}} # These concrete targets require flat arrays with no NULL elements. v.oid == 1021 && return convert(S, parse_array(text, Float32)::Vector{Float32}) v.oid == 1022 && return convert(S, parse_array(text, Float64)::Vector{Float64}) diff --git a/test/postgres_trim_queries.jl b/test/postgres_trim_queries.jl index aa3faaf..39431c1 100644 --- a/test/postgres_trim_queries.jl +++ b/test/postgres_trim_queries.jl @@ -38,6 +38,7 @@ struct TrimCompatibility absent_int::Union{Missing, Int64} widened::Float64 integers::Vector{Int64} + octets::Vector{UInt8} widened_array::Vector{Float64} clock::Time maybe_clock::Union{Nothing, Time} @@ -93,6 +94,7 @@ function run_postgres_trim_queries()::Nothing SELECT 1e15::float4 AS float4_int, 1e15::float8 AS float8_int, NULL::float8 AS absent_int, 0.1::float4 AS widened, ARRAY[1e15::float4, -1e15::float4] AS integers, + ARRAY[100::float4] AS octets, ARRAY[0.1::float4] AS widened_array, '1969-12-31 23:59:59.999999'::timestamp AS clock, '1800-01-01 00:00:00.123456+00'::timestamptz AS maybe_clock, @@ -104,6 +106,7 @@ function run_postgres_trim_queries()::Nothing _check(compat.absent_int === missing, "unexpected nullable integer") _check(compat.widened === Float64(0.1f0) && compat.widened_array == [Float64(0.1f0)], "lost float4 precision") _check(compat.integers == Int64[999999986991104, -999999986991104], "unexpected exponent array conversion") + _check(compat.octets == UInt8[100], "unexpected unsigned array conversion") _check(compat.clock === Time(23, 59, 59, 999, 999) && compat.clocks == [compat.clock], "lost timestamp time precision") _check(compat.maybe_clock === Time(0, 0, 0, 123, 456) && compat.absent_clock === missing, "unexpected timestamp offset or NULL") _check(compat.json_null === "null" && compat.sql_null === nothing, "confused JSON null with SQL NULL") diff --git a/test/result_decoding.jl b/test/result_decoding.jl index 10d261e..ce3851c 100644 --- a/test/result_decoding.jl +++ b/test/result_decoding.jl @@ -32,6 +32,8 @@ function test_typed_read_compatibility(conn) end @test DBInterface.execute(conn, "SELECT ARRAY[1e15::$pgtype, -1e15::$pgtype] AS value", (), NamedTuple{(:value,), Tuple{Vector{Int64}}}).value == [Int64(decoded), -Int64(decoded)] + @test DBInterface.execute(conn, "SELECT ARRAY[100::$pgtype] AS value", (), + NamedTuple{(:value,), Tuple{Vector{UInt8}}}).value == UInt8[100] end widened = DBInterface.execute(conn, "SELECT 0.1::float4 AS value", (), NamedTuple{(:value,), Tuple{Float64}}).value @test widened === Float64(0.1f0) @@ -200,6 +202,9 @@ function test_result_decoding() @test_throws InexactError decoding_field(Int8, "1.28e+2", oid) @test decoding_field(Cuint, "1e+2", oid) === Cuint(100) @test_throws InexactError decoding_field(Cuint, "-1e+2", oid) + @test decoding_field(Vector{UInt8}, "{1e+2}", array_oid) == UInt8[100] + @test_throws InexactError decoding_field(Vector{UInt8}, "{2.56e+2}", array_oid) + @test_throws InexactError decoding_field(Vector{UInt8}, "{-1e+2}", array_oid) for T in (Vector{Int64}, Union{Nothing, Vector{Int64}}, Union{Missing, Vector{Int64}}) @test decoding_field(T, "{1e+15,-1e+15}", array_oid) == Int64[parse(F, "1e+15"), parse(F, "-1e+15")] end From 0cea219375a77f5e42b95c59ff195fa5e7b25cd3 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Tue, 27 Oct 2026 13:58:50 -0600 Subject: [PATCH 8/8] Review fixes: byte vectors from integer arrays, docs, test runner A Vector{UInt8} field over an integer array column decoded the array text as bytea, so {1,2} read as the bytes of "{1,2}". It converts element by element again, as on main. docs/src/support.md still said --trim was unsupported; it now says what the trim test checks and why --trim=safe still fails on stock Julia. Date parameters pad with string(n; pad), the trim test runner uses a Timer instead of a polling loop, and the live typed-read test drops session settings that do not affect its values. Version 2.4.0: main already released 2.3.1, and typed reads change behavior. Co-Authored-By: Claude Opus 5.5 --- Project.toml | 2 +- docs/src/manual.md | 2 +- docs/src/support.md | 10 +++++-- src/api/types.jl | 27 ++++++++++--------- src/execute.jl | 13 +++------ test/result_decoding.jl | 7 +++-- test/trim_compile_tests.jl | 54 +++++++++++++------------------------- 7 files changed, 50 insertions(+), 65 deletions(-) diff --git a/Project.toml b/Project.toml index b59c721..3ba4576 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Postgres" uuid = "8f23287e-300e-4f50-bc2b-9f1dfe95da84" -version = "2.3.1" +version = "2.4.0" [deps] ConcurrentUtilities = "f0e56b4a-5159-44fe-b623-3e5288b988bb" diff --git a/docs/src/manual.md b/docs/src/manual.md index 437a004..0056c5a 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -119,7 +119,7 @@ profiles = DBInterface.execute(conn, """ The `postgres=(name=:column_name,)` tag is only needed when a column should map to a differently named field. Columns such as `id` or `name` can be left untagged because they already match the Julia field name. -With the default `Postgres.PostgresStyle`, a field whose type Postgres.jl can parse from text (strings, symbols, enums, integers, floats, `Bool`, `Char`, `Date`, `Time`, `DateTime`, `UUID`, `Vector{UInt8}`, lazy JSON, timestamps, decimals, and vectors of these) is decoded from the column text by its declared type, optionally wrapped in `Union{Nothing, …}` or `Union{Missing, …}`. A `String` field receives any column's text, so a `json` or `jsonb` column can be read as its JSON text. An integer field also accepts integral `numeric` values such as `12.00`. A parser registered with `register_type!` still applies when it returns the field's type. Typed reads of these fields compile to static calls, so they work in `juliac --trim` builds. Other field types, and all fields read with a custom driver style, receive the value decoded by the column's type, so custom-style `lift` methods see decoded values. +With the default `Postgres.PostgresStyle`, fields of common types are parsed from the column text by their declared type: strings, symbols, enums, integers, floats, `Bool`, `Char`, `Date`, `Time`, `DateTime`, `UUID`, `Vector{UInt8}`, lazy JSON, timestamps, decimals, vectors of these, and any of them in a `Union` with `Nothing` or `Missing`. A `String` field receives any column's text, so a `json` column reads as its JSON text. Integer fields accept integral `numeric` values such as `12.00`, and `Date` and `Time` fields over timestamp columns get the UTC date or time of day. A parser registered with `register_type!` still applies when it returns the field's type. These reads need no runtime dispatch, which `juliac --trim` builds require. Other field types, and fields read with a custom driver style, receive the value decoded for the column type. ### Driver Styles diff --git a/docs/src/support.md b/docs/src/support.md index 5e29418..3fd028c 100644 --- a/docs/src/support.md +++ b/docs/src/support.md @@ -108,5 +108,11 @@ represented as a Boolean and raise an error. Select them as text, for example ## Native Compilation -JuliaC `--trim` compilation is not supported in Postgres.jl. Normal Julia -package precompilation is supported and is part of CI. +JuliaC `--trim` builds work with `--trim=unsafe-warn`. Postgres.jl itself has +no trim verifier errors on Julia 1.12 and later, but it reaches parts of +released Julia's Base64 (SCRAM authentication) and `Dates.CompoundPeriod` +(interval decoding) that are not trim-safe, so `--trim=safe` rejects the build. +CI compiles a workload of connections, prepared statements, transactions, +parameters, and typed reads, and runs it against PostgreSQL with trust +authentication. Typed reads into general unions, and reads with a custom driver +style, use runtime dispatch. diff --git a/src/api/types.jl b/src/api/types.jl index 1bdab37..4a89850 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -410,15 +410,15 @@ end # timestamp(tz) column into a Date field: the UTC date, as for DateTime fields pg_parse_date_any(s::AbstractString)::Date = occursin(':', s) ? Date(pg_parse_datetime_any(s)) : pg_parse_date(s) -# Timestamp columns contribute their UTC time of day without losing microseconds. +# timestamp(tz) column into a Time field: the UTC time of day function pg_parse_time_any(s::AbstractString)::Time + # "infinity" has no space and would otherwise reach the time parser _check_temporal_special(s, "timestamp") return occursin(' ', s) ? convert(Time, pg_parse_timestamp(s)) : pg_parse_time(s) end -# Integer fields also read integral numeric text such as "12.00". Widening to -# Int128 detects values outside the target range and throws InexactError; -# text outside Int128's range or in other forms throws the target parser's error. +# Integer fields also read integral numeric text such as "12.00". Out-of-range +# values throw InexactError; other non-integer text throws the parser's error. function pg_parse_integer(::Type{T}, s::AbstractString) where {T <: Integer} x = Parsers.tryparse(T, s) x === nothing || return x @@ -1000,12 +1000,10 @@ _wire_field(::Type{<:AbstractVector{T}}) where {T} = T <: WireFieldScalar _field_source(::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = _wire_field(T) ? _field_text(v.text) : parse_value(v.oid, v.text, v.registry) -# With the default style, field types that the lifts below (and StructUtils' -# own String, Symbol and Enum lifts) parse from text decode from the column -# text. Floating columns retain their precision for numeric targets. These -# sources have inferred types under `--trim`; OID decoding returns `Any`. -# Custom styles keep the OID decoder so their lifts still receive decoded values, -# and other field types keep it too. +# With the default style, fields of these types (and enums) are lifted from the +# column text, so typed reads infer and compile under `--trim`; `parse_value` +# returns `Any`. Other field types, general unions, and custom styles (whose +# lifts expect decoded values) keep the OID decoder. const TextField = Union{String, Symbol, Bool, Char, Int8, Int16, Int32, Int64, Cuint, Float32, Float64, Date, Time, DateTime, UUID, Vector{UInt8}, JSONType, Vector{String}, Vector{Int16}, Vector{Int32}, Vector{Int64}, Vector{Float32}, Vector{Float64}, Vector{Bool}, Vector{Date}, Vector{Time}, @@ -1023,15 +1021,18 @@ function _field_source(::PostgresStyle, ::Type{T}, v::FieldValue) where {T} x isa S && return x end text = _field_text(v.text) - # Conversions must use the column's floating precision, including exponent - # output. Parsing float4 directly as Float64 would silently change its value. + # Parse float columns at their own precision, then convert: large values + # print in exponent form ("1e+15"), and float4 widened to Float64 keeps its + # Float32 value. if S <: Union{Signed, Unsigned, AbstractFloat} v.oid == 700 && return convert(S, Parsers.parse(Float32, text)) v.oid == 701 && return convert(S, Parsers.parse(Float64, text)) elseif S <: Union{Vector{<:Signed}, Vector{<:Unsigned}, Vector{<:AbstractFloat}} - # These concrete targets require flat arrays with no NULL elements. v.oid == 1021 && return convert(S, parse_array(text, Float32)::Vector{Float32}) v.oid == 1022 && return convert(S, parse_array(text, Float64)::Vector{Float64}) + # the Vector{UInt8} lift decodes bytea text; other arrays convert by element + S === Vector{UInt8} && v.oid != 17 && info !== nothing && info.julia_type <: AbstractVector && + return convert(S, parse_array(text, Int64)::Vector{Int64}) end return text end diff --git a/src/execute.jl b/src/execute.jl index 8c2b72d..601f53f 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -262,17 +262,10 @@ Base.close(cursor::Cursor) = DBInterface.close!(cursor) _param(x::AbstractString)::String = String(x) _param(x)::String = string(x) # Same text as `string(x)`, whose Dates formatting is not trim-safe. -function _padded(n::Integer, width::Int) - digits = string(abs(n)) - for _ in ncodeunits(digits)+1:width - digits = string('0', digits) # lpad is not trim-safe - end - return n < 0 ? string('-', digits) : digits -end -_param(x::Date)::String = string(_padded(year(x), 4), '-', _padded(month(x), 2), '-', _padded(day(x), 2)) +_param(x::Date)::String = string(string(year(x); pad=4), '-', string(month(x); pad=2), '-', string(day(x); pad=2)) function _param(x::DateTime)::String - text = string(_param(Date(x)), 'T', _padded(hour(x), 2), ':', _padded(minute(x), 2), ':', _padded(second(x), 2)) - return iszero(millisecond(x)) ? text : string(text, '.', _padded(millisecond(x), 3)) + text = string(_param(Date(x)), 'T', string(hour(x); pad=2), ':', string(minute(x); pad=2), ':', string(second(x); pad=2)) + return iszero(millisecond(x)) ? text : string(text, '.', string(millisecond(x); pad=3)) end function _param(x::Durations.Timestamp)::String # PostgreSQL would round finer fractions. Require an exact conversion. diff --git a/test/result_decoding.jl b/test/result_decoding.jl index ce3851c..5296817 100644 --- a/test/result_decoding.jl +++ b/test/result_decoding.jl @@ -13,9 +13,9 @@ end function test_typed_read_compatibility(conn) @testset "Typed read compatibility on PostgreSQL" begin + # settings that change how the float and timestamp values below print settings = ("SET LOCAL TimeZone = 'UTC'", "SET LOCAL TimeZone = 'Asia/Kolkata'", - "SET LOCAL TimeZone = 'America/New_York'", "SET LOCAL IntervalStyle = 'iso_8601'", - "SET LOCAL bytea_output = 'escape'", "SET LOCAL extra_float_digits = 0", + "SET LOCAL TimeZone = 'America/New_York'", "SET LOCAL extra_float_digits = 0", "SET LOCAL extra_float_digits = 3") for setting in settings DBInterface.transaction(conn) do @@ -179,6 +179,9 @@ function test_result_decoding() target = NamedTuple{Tuple(names), Tuple{Int64, Int32, Date, Vector{Int64}, Union{Nothing, Int64}, DecodingColor}} typed = StructUtils.make(target, row, Postgres.PostgresStyle()) @test typed == (total=12, count=Int32(7), day=Date(2024, 1, 3), amounts=[12, 3], maybe=5, color=decoding_red) + # byte vectors decode bytea text, but integer arrays by element + @test decoding_field(Vector{UInt8}, "\\x0102", 17) == UInt8[1, 2] + @test decoding_field(Vector{UInt8}, "{1,2}", 1007) == UInt8[1, 2] @test_throws ArgumentError StructUtils.make(NamedTuple{(:total,), Tuple{Int64}}, decoding_row(["12.50"], [1700]; names=[:total]), Postgres.PostgresStyle()) # a registered decoder applies when it produces the field's type; diff --git a/test/trim_compile_tests.jl b/test/trim_compile_tests.jl index 79285e1..fe118fb 100644 --- a/test/trim_compile_tests.jl +++ b/test/trim_compile_tests.jl @@ -1,7 +1,6 @@ -# Compiles test/postgres_trim_queries.jl with juliac, requires a clean verifier -# report apart from known stdlib gaps, and runs the executable against the test -# database. `--trim=unsafe-warn` builds despite reported errors. Executing this -# workload verifies its runtime paths, not strict `--trim=safe` support. +# Compiles test/postgres_trim_queries.jl with JuliaC and runs the executable +# against the test database. Any verifier error outside the known stdlib gaps +# fails the test; `--trim=unsafe-warn` lets the build finish despite those gaps. import Pkg const _TRIM_SUPPORTED = VERSION >= v"1.12.0-rc1" && isempty(VERSION.prerelease) @@ -24,36 +23,26 @@ function _prepare_trim_project(trim_project::String)::Nothing end function _run_with_timeout(cmd::Cmd; timeout_s::Float64, label::String) - # JuliaC and its compiler subprocess share these streams. A pipe keeps - # their writes ordered without competing regular-file offsets. + # JuliaC and its compiler subprocess write to one pipe so their output stays + # complete and in order pipe = Pipe() - captured = IOBuffer() - exit_code, timed_out = -1, false proc = run(pipeline(ignorestatus(cmd), stdout = pipe, stderr = pipe); wait = false) close(pipe.in) - reader = @async try - while !eof(pipe) - write(captured, readavailable(pipe)) - end - catch err - timed_out || rethrow() + timed_out = Ref(false) + timer = Timer(timeout_s) do _ + timed_out[] = true + kill(proc) + close(pipe) end - started_at = time() - while Base.process_running(proc) - if time() - started_at >= timeout_s - timed_out = true - kill(proc) - break - end - sleep(0.1) + captured = IOBuffer() + while !eof(pipe) + write(captured, readavailable(pipe)) end - timed_out ? close(pipe) : wait(proc) - wait(reader) - close(pipe) - exit_code = something(proc.exitcode, -1) + close(timer) output = String(take!(captured)) - timed_out && error("trim $(label) timed out after $(timeout_s)s:\n$(output)") - return exit_code, output + timed_out[] && error("trim $(label) timed out after $(timeout_s)s:\n$(output)") + wait(proc) + return proc.exitcode, output end # Released Julia's Base64 pipes and Dates.CompoundPeriod constructor are not @@ -72,7 +61,7 @@ end function _trim_verify_counts(output::String) blocks = split("\n" * output, r"\n(?:Verifier error|Error) #\d+:")[2:end] gaps = count(_stdlib_trim_gap, blocks) - warnings = length(collect(eachmatch(r"(?m)^(?:Verifier warning|Warning) #\d+:", output))) + warnings = count(r"(?m)^(?:Verifier warning|Warning) #\d+:", output) summary = match(r"Trim verify finished with (\d+) errors?, (\d+) warnings?", output) if summary !== nothing reported = parse(Int, summary[1]) + parse(Int, summary[2]) @@ -95,13 +84,6 @@ function test_trim_verifier_diagnostics() @test _trim_verify_counts("Compiling...\n") == (0, 0, 0) @test _trim_verify_counts("Error #1: unresolved call$package_frame\nTrim verify finished with 1 error, 0 warnings.") == (1, 0, 0) @test_throws ErrorException _trim_verify_counts("Trim verify finished with 1 error, 0 warnings.") - - julia = joinpath(Sys.BINDIR, Base.julia_exename()) - child = `$julia --startup-file=no -e $("print(stderr, repeat(\"x\", 65536)); flush(stderr)")` - script = "print(stderr, \"start\\n\"); flush(stderr); run(pipeline($(repr(child)); stdout, stderr)); print(stderr, \"end\\n\"); flush(stderr)" - code, output = _run_with_timeout(`$julia --startup-file=no -e $script`; timeout_s=30.0, label="capture test") - @test code == 0 - @test output == "start\n" * repeat("x", 65536) * "end\n" end end