Replace the cereal library with our own encoding and decoding functions

BENCHMARKS.md

@@ -28,31 +28,36 @@ The second column is wall clock time in seconds, the third is peak heap memory a
 ### Materializing 100_000 rows with 13 columns each into a List of Records
 
 This runs with 2 concurrent queries, 10 times over:
+
+This benchmark is unfair towards both hpgsql and postgresql-simple because the row decoder is Generically derived for them while it is hand-written for hasql.
 ```csv
-postgresql-simple Record List (100000 rows),11.90,142.65M,101.6
-hasql Record List (100000 rows),6.279,142.48M,78.0
-hpgsql Record List (100000 rows),3.886,72.07M,98.2
+postgresql-simple Record List (100000 rows),12.10,142.65M,120.5
+hasql Record List (100000 rows),6.314,142.48M,77.8
+hpgsql Record List (100000 rows),4.092,72.07M,119.7
 ```
 
 ### Materializing 100_000 rows with 13 columns each into a List of Tuples
 
 This runs with 2 concurrent queries, 10 times over:
 ```csv
-postgresql-simple Tuple List (100000 rows),15.03,142.78M,149.8
-hasql Tuple List (100000 rows),8.376,142.48M,195.4
-hpgsql Tuple List (100000 rows),4.689,72.07M,149.4
+postgresql-simple Tuple List (100000 rows),14.74,142.52M,144.9
+hasql Tuple List (100000 rows),8.263,142.48M,202.2
+hpgsql Tuple List (100000 rows),4.648,72.07M,150.4
 ```
 
 ### Streaming 100_000 rows with 13 columns as Records
 
 This runs with 2 concurrent queries, 10 times over.
-Hpgsql's implementation streams directly from the socket while the others use cursors, so
+
+This benchmark is unfair towards both hpgsql and postgresql-simple because the row decoder is Generically derived for them while it is hand-written for hasql.
+
+However, Hpgsql's implementation streams directly from the socket while the others use cursors, so
 it might not be a fair comparison in terms of implementation (e.g. you can advance multiple
 cursors simultaneously, but not hpgsql's Streamed-from-socket streams).
 ```csv
-streaming-postgresql-simple Record Stream (100000 rows),16.69,73.24M,0.1
-postgresql-simple Record fold (100000 rows),13.39,78.29M,0.2
-hpgsql Record Stream (100000 rows),1.457,72.07M,0.2
+streaming-postgresql-simple Record Stream (100000 rows),13.32,73.41M,0.0
+postgresql-simple Record fold (100000 rows),13.47,77.81M,0.0
+hpgsql Record Stream (100000 rows),1.421,72.07M,0.0
 ```
 
 ### Streaming 100_000 rows with 13 columns as Tuples
@@ -62,16 +67,16 @@ Hpgsql's implementation streams directly from the socket while the others use cu
 it might not be a fair comparison in terms of implementation (e.g. you can advance multiple
 cursors simultaneously, but not hpgsql's Streamed-from-socket streams).
 ```csv
-streaming-postgresql-simple Tuple Stream (100000 rows),13.89,73.28M,0.1
-postgresql-simple Tuple fold (100000 rows),13.69,81.99M,0.2
-hpgsql Tuple Stream (100000 rows),1.076,72.07M,0.2
+streaming-postgresql-simple Tuple Stream (100000 rows),14.10,73.26M,0.0
+postgresql-simple Tuple fold (100000 rows),13.45,84.42M,0.0
+hpgsql Tuple Stream (100000 rows),1.025,72.07M,0.0
 ```
 
 ### COPY FROM STDIN
 
 This compares hpgsql's binary copy to a `forM` loop writing text rows.
 
 ```csv
-postgresql-simple text COPY (100000 rows),1.353,72.10M,3.9
-hpgsql copyFromS binary COPY (100000 rows),1.239,72.07M,11.0
+postgresql-simple text COPY (100000 rows),1.348,72.10M,3.8
+hpgsql copyFromS binary COPY (100000 rows),672.1,72.07M,10.8
 ```

README.md

@@ -51,11 +51,11 @@ You should start by swapping all of "postgresql-simple", "postgresql-libpq", and
 
 ## Performance
 
-Some benchmarks show materializing large query results with hpgsql takes 31-33% the time postgresql-simple takes, and 56-62% the time hasql takes (on my computer, Linux x64, GHC 9.10.3, compiled with -O1).
+Some benchmarks show materializing large query results with hpgsql takes 31-34% the time postgresql-simple takes, and 55-65% the time hasql takes (on my computer, Linux x64, GHC 9.10.3, compiled with -O1).
 
 When comparing hpgsql's Stream querying, hpgsql takes 9-11% the time of both [streaming-postgresql-simple](https://hackage.haskell.org/package/streaming-postgresql-simple) and postgresql-simple's cursor folding functions, although this might not be a fair comparison for some use cases.
 
-hpgsql's binary COPY runs in about 92% the time of postgresql-simple's textual COPY.
+hpgsql's binary COPY runs in ~50% the time of postgresql-simple's textual COPY.
 
 Peak allocated memory is harder to analyze.
 

hpgsql-benchmarks/src/Main.hs

@@ -154,7 +154,13 @@ main = do
 
   putStrLn "IMPORTANT: all measurements collected over 10 runs of each benchmark"
   when (numConcurrentConnections > 1) $ putStrLn $ "IMPORTANT: all benchmarks except COPY involve running the benchmarked query in " ++ show numConcurrentConnections ++ " connections in parallel"
+
+  -- Warm up postgres with a generate_series query and GC before tests
+  warmupConn <- hpgsqlConnect
+  void $ Hpgsql.execute warmupConn "SELECT * FROM generate_series(1,100000)"
+  Hpgsql.Connection.closeGracefully warmupConn
   performBlockingMajorGC
+
   statsBefore <- getRTSStats
   hspecWith defaultConfig {configFormat = Just (formatterToFormat silent)} $ do
     describe "Parsing 13-column rows into a List" $ do

hpgsql-tests/CopySpec.hs

@@ -2,7 +2,7 @@ module CopySpec where
 
 import Control.Monad (forM_)
 import Control.Monad.IO.Class (liftIO)
-import Data.Int (Int32)
+import Data.Int (Int32, Int64)
 import Data.Text (Text)
 import qualified Data.Text as Text
 import qualified Data.Text.Encoding as TE
@@ -38,36 +38,37 @@ spec = do
       "putCopyError"
       copyError
 
-genRows :: Gen.Gen [(Int32, Text)]
+genRows :: Gen.Gen [(Int32, Text, Int64)]
 genRows = do
   numRows <- Gen.int (Gen.linear 0 1000)
   names <- Gen.list (Gen.singleton numRows) $ Gen.text (Gen.linear 1 50) Gen.alphaNum
-  pure $ zip [1 ..] names
+  numbers <- Gen.list (Gen.singleton numRows) $ Gen.int64 (Gen.linear (-100) 100)
+  pure $ zip3 [1 ..] names numbers
 
 copyTextFmtStatementSucceeding :: HPgConnection -> PropertyT IO ()
 copyTextFmtStatementSucceeding conn = hedgehog $ do
   rows <- Gen.forAll genRows
   result <- liftIO $ withRollback conn $ do
-    execute_ conn "CREATE UNLOGGED TABLE copy_test0 (id INT NOT NULL, name TEXT NOT NULL)"
+    execute_ conn "CREATE UNLOGGED TABLE copy_test0 (id INT NOT NULL, name TEXT NOT NULL, some_num BIGINT)"
     withCopy_
       conn
       "COPY copy_test0 FROM STDIN WITH (FORMAT CSV);"
-      ( forM_ rows $ \(eid, ename) ->
-          putCopyData conn $ TE.encodeUtf8 $ Text.pack (show eid) <> "," <> ename <> "\n"
+      ( forM_ rows $ \(eid, ename, somenum) ->
+          putCopyData conn $ TE.encodeUtf8 $ Text.pack (show eid) <> "," <> ename <> "," <> Text.pack (show somenum) <> "\n"
       )
-    query conn "SELECT id, name FROM copy_test0 ORDER BY id"
+    query conn "SELECT id, name, some_num FROM copy_test0 ORDER BY id"
   result === rows
 
 copyBinaryFmtStatementSucceeding :: HPgConnection -> PropertyT IO ()
 copyBinaryFmtStatementSucceeding conn = hedgehog $ do
   rows <- Gen.forAll genRows
   result <- liftIO $ withRollback conn $ do
-    execute_ conn "CREATE UNLOGGED TABLE copy_test1 (id INT NOT NULL, name TEXT NOT NULL)"
+    execute_ conn "CREATE UNLOGGED TABLE copy_test1 (id INT NOT NULL, name TEXT NOT NULL, some_num BIGINT)"
     copyFrom
       conn
       "COPY copy_test1 FROM STDIN WITH (FORMAT BINARY);"
       rows
-    query conn "SELECT id, name FROM copy_test1 ORDER BY id"
+    query conn "SELECT id, name, some_num FROM copy_test1 ORDER BY id"
   result === rows
 
 copyError :: HPgConnection -> IO ()

hpgsql-tests/EncodingDecodingSpec.hs

@@ -31,6 +31,7 @@ import qualified Data.Vector as Vector
 import DbUtils
   ( aroundConn,
     irrecoverableErrorWithMsgAndStmt,
+    testConnInfo,
     withRollback,
   )
 import GHC.Float (float2Double)
@@ -40,7 +41,7 @@ import qualified Hedgehog as Gen
 import qualified Hedgehog.Gen as Gen
 import qualified Hedgehog.Range as Gen
 import Hpgsql
-import Hpgsql.Connection (refreshTypeInfoCache)
+import Hpgsql.Connection (ConnectOpts (..), connect, connectOpts, defaultConnectOpts, refreshTypeInfoCache, withConnectionOpts)
 import Hpgsql.Encoding (EncodingContext (..), FieldDecoder (..), FieldEncoder (..), FieldInfo (..), FromPgField (..), FromPgRow (..), LowerCasedPgEnum (..), RowEncoder (..), ToPgField (..), ToPgRow (..), compositeTypeDecoder, compositeTypeEncoder, nullableField, rawBytesFieldDecoder, singleField, typeFieldDecoder, typeFieldEncoder, typeMustBeNamed, typeOidWithName)
 import Hpgsql.Pipeline (pipeline, pipelineWith, runPipeline)
 import Hpgsql.Query (mkQuery, sql, vALUES)
@@ -156,16 +157,22 @@ spec = parallel $ do
     it
       "Generically derived types round-trip"
       queryGenericallyDerivedTypesRoundTrip
+  it
+    "0-columns results can be decoded"
+    zeroColumnsResults
+
+zeroColumnsResults :: IO ()
+zeroColumnsResults = do
+  hpgsqlConnInfo <- testConnInfo
+  -- This test is important to test the "slow" decoding path of `decodeDataRow`
+  -- in BinarySerializer.hs. The number of rows needs to be a bit large
+  -- and the recvChunkSize pretty small for that code path to be exercised,
+  -- as per some debug printing.
+  withConnectionOpts defaultConnectOpts {recvChunkSize = 5} hpgsqlConnInfo 10 $ \conn -> do
+    execute conn "SELECT FROM generate_series(1,601)" `shouldReturn` 601
 
 valuesRoundTrip :: HPgConnection -> IO ()
 valuesRoundTrip conn = do
-  -- TODO: Property-based test to generate the values
-  -- TODO: Include NULLs
-  -- TODO: Test +-infinity for types where we can
-  -- TODO: Test all types in the regions of values close to `minBound`, 0, and `maxBound`
-  -- TODO: Test floats, timestamptz and other very granular but discrete type in the regions of values
-  --       close to `minBound`, 0, and `maxBound`, with e.g. microsecond precision/fractional values
-  -- TODO: Test +-Infinity and NaN for floats and doubles
   let row = ((-49) :: Int, False :: Bool, 2 :: Int16, 3 :: Int32, fromGregorian 1900 02 28, 42 :: Int64, UTCTime (fromGregorian 1999 12 31) 0, '意' :: Char, '&' :: Char, CalendarDiffTime 3 86403, Aeson.Null)
   queryWith rowDecoder conn (mkQuery "SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11" row) `shouldReturn` [row]
 

hpgsql/hpgsql.cabal

@@ -43,6 +43,7 @@ library
       Hpgsql.Types
   other-modules:
       Hpgsql.Base
+      Hpgsql.Encoding.BinarySerializer
       Hpgsql.Internal
       Hpgsql.LanguageHaskell.FromThExtension
       Hpgsql.LanguageHaskell.GhcParserOpts
@@ -102,7 +103,6 @@ library
     base >= 4.18 && < 4.22,
     bytestring >= 0.11 && < 0.13,
     case-insensitive >= 1.2 && < 1.3,
-    cereal >= 0.5 && < 0.6,
     containers >= 0.6 && < 0.8,
     crypton >= 1.0.0 && < 1.1,
     memory >= 0.18.0 && < 0.19,

hpgsql/src/Hpgsql/Builder.hs

@@ -1,12 +1,10 @@
-module Hpgsql.Builder where
-
--- \| This module replicates parts of the API of Data.ByteString.Builder but its own
+-- | This module replicates parts of the API of Data.ByteString.Builder but its own
 -- builder is length-aware, which makes other parts of the code a little bit nicer.
 -- In COPY benchmarks, this module was introduced in a commit (together with other
 -- changes, like replacing `Maybe` with `BinaryField` in `ToPgField`) that barely
 -- changed memory usage and runtime.
 -- The benefits are exclusively for code readability, then.
--- \|
+module Hpgsql.Builder where
 
 import Data.ByteString (ByteString)
 import qualified Data.ByteString as BS
@@ -23,7 +21,9 @@ instance Show BinaryField where
   show SqlNull = "NULL"
   show (NotNull bs) = show bs
 
-data LengthAwareBuilder = LengthAwareBuilder !Int32 !Builder.Builder
+-- | The lazy (instead of strict/with a bang) Builder (second arg) makes
+-- our copyFromS benchmark run ~4.3% faster and allocate ~3.8% less total memory.
+data LengthAwareBuilder = LengthAwareBuilder !Int32 Builder.Builder
 
 type Builder = LengthAwareBuilder
 

hpgsql/src/Hpgsql/Connection.hs

@@ -8,6 +8,7 @@ module Hpgsql.Connection
     closeForcefully,
     connectionIsClosed,
     ConnectionString (..),
+    ConnectOpts (..),
     parseLibpqConnectionString,
     ResetConnectionOpts (..),
     resetConnectionState,
@@ -48,7 +49,7 @@ import Data.Text (Text)
 import qualified Data.Text as Text
 import Data.Text.Encoding (encodeUtf8)
 import Hpgsql.Internal (closeForcefully, closeGracefully, connect, connectOpts, connectionIsClosed, defaultConnectOpts, getBackendPid, getParameterStatus, refreshTypeInfoCache, resetConnectionState, resetTypeInfoCache, withConnection, withConnectionOpts)
-import Hpgsql.InternalTypes (ConnectionString (..), ResetConnectionOpts (..))
+import Hpgsql.InternalTypes (ConnectOpts (..), ConnectionString (..), ResetConnectionOpts (..))
 import Network.URI
   ( URI (..),
     URIAuth (..),