Backpatch specialized row decoders without breaking API changes

Runfile

@@ -76,7 +76,7 @@ tests:
         if [ -n "$NIX" ]; then
             nix-build --no-out-link -A "testsPg${pg}" --argstr hspecArgs "$TARGS"
         else
-            cabal build hpgsql-tests hpgsql-simple-compat-tests
+            cabal build hpgsql-tests # hpgsql-simple-compat-tests
             nix-shell -A "shellPg${pg}" ./default.nix --run "./scripts/run-tests-db-internal.sh $TARGS"
         fi
     done

TODO.md

@@ -0,0 +1,8 @@
+- Test both `singleField fieldDecoder` and `singleFieldRowDecoder` for every type in our tests.
+  - For array types too we have more than one arrayField* variant
+- Some types might still not derive specialized row decoders
+  - And rewrite rules too are missing for many
+- Text internals usage.. is it safe? Double-check.
+- Check that we're not holding on to internal buffers when Record fields being materialized into aren't strict
+- Write property-based tests for PinnedByteArray functions
+- Double check that for this PR the inlined and not inlined versions really differ in performance

csharp-benchmarks/Program.cs

@@ -103,9 +103,9 @@ string FormatSecs(double s)
            'item-' || g::text,
            g::float8 * 1.5,
            g::float8 * 2.5,
-           NULL::int4,
-           NULL::text,
-           NULL::float8,
+           (CASE WHEN g%2=1 THEN NULL ELSE g::int4 END),
+           (CASE WHEN g%2=1 THEN NULL ELSE '' END)::text,
+           (CASE WHEN g%2=1 THEN NULL ELSE 0.0 END)::float8,
            NULL::date,
            g::numeric,
            g::float4,

hpgsql-benchmarks/src/Main.hs

@@ -47,6 +47,7 @@ import Hpgsql.Connection (renderLibpqConnectionString)
 import qualified Hpgsql.Connection
 import qualified Hpgsql.Connection as Hpgsql
 import qualified Hpgsql.Copy
+import Hpgsql.Encoding (inlinedFieldRowDecoder)
 import qualified Hpgsql.Encoding as Hpgsql
 import qualified Hpgsql.Query as Hpgsql
 import qualified Hpgsql.Types as Hpgsql
@@ -91,6 +92,14 @@ data BenchRow = BenchRow
   deriving stock (Generic, Show, Eq)
   deriving anyclass (NFData, Hpgsql.FromPgRow, PGSimple.FromRow)
 
+singleFieldBenchRowDecoder :: Hpgsql.RowDecoder BenchRow
+singleFieldBenchRowDecoder =
+  BenchRow <$> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder <*> Hpgsql.singleField Hpgsql.fieldDecoder
+
+fullyInlinedBenchRowDecoder :: Hpgsql.RowDecoder BenchRow
+fullyInlinedBenchRowDecoder =
+  BenchRow <$> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder <*> inlinedFieldRowDecoder
+
 data HasqlBenchRow = HasqlBenchRow
   { hbrId :: !Int32,
     hbrDate1 :: !Day,
@@ -183,10 +192,10 @@ main = do
   statsBefore <- getRTSStats
   hspecWith defaultConfig {configFormat = Just (formatterToFormat silent)} $ do
     let n :: Int = 100_000
-    let sql17 = "SELECT g, ('2000-01-01'::date + g::int4), ('2000-06-15'::date + g::int4), ('2000-01-01T00:00:00Z'::timestamptz + g * interval '1 second'), ('2020-06-15T12:00:00Z'::timestamptz + g * interval '1 minute'), 'row-' || g::text, 'item-' || g::text, g::float8 * 1.5, g::float8 * 2.5, NULL::int4, NULL::text, NULL::float8, NULL::date, g::numeric, g::float4, g%2=0, g%2=1 FROM generate_series(1,$1) g"
-        sql17Simple = "SELECT g, ('2000-01-01'::date + g::int4), ('2000-06-15'::date + g::int4), ('2000-01-01T00:00:00Z'::timestamptz + g * interval '1 second'), ('2020-06-15T12:00:00Z'::timestamptz + g * interval '1 minute'), 'row-' || g::text, 'item-' || g::text, g::float8 * 1.5, g::float8 * 2.5, NULL::int4, NULL::text, NULL::float8, NULL::date, g::numeric, g::float4, g%2=0, g%2=1 FROM generate_series(1,?) g"
-        sql13 = "SELECT g, ('2000-01-01'::date + g::int4), ('2000-06-15'::date + g::int4), ('2000-01-01T00:00:00Z'::timestamptz + g * interval '1 second'), ('2020-06-15T12:00:00Z'::timestamptz + g * interval '1 minute'), 'row-' || g::text, 'item-' || g::text, g::float8 * 1.5, g::float8 * 2.5, NULL::int4, NULL::text, NULL::float8, NULL::date FROM generate_series(1,$1) g"
-        sql13Simple = "SELECT g, ('2000-01-01'::date + g::int4), ('2000-06-15'::date + g::int4), ('2000-01-01T00:00:00Z'::timestamptz + g * interval '1 second'), ('2020-06-15T12:00:00Z'::timestamptz + g * interval '1 minute'), 'row-' || g::text, 'item-' || g::text, g::float8 * 1.5, g::float8 * 2.5, NULL::int4, NULL::text, NULL::float8, NULL::date FROM generate_series(1,?) g"
+    let sql17 = "SELECT g, ('2000-01-01'::date + g::int4), ('2000-06-15'::date + g::int4), ('2000-01-01T00:00:00Z'::timestamptz + g * interval '1 second'), ('2020-06-15T12:00:00Z'::timestamptz + g * interval '1 minute'), 'row-' || g::text, 'item-' || g::text, g::float8 * 1.5, g::float8 * 2.5, (CASE WHEN g%2=1 THEN NULL ELSE g::int4 END), (CASE WHEN g%2=1 THEN NULL ELSE '' END)::text, (CASE WHEN g%2=1 THEN NULL ELSE 0.0 END)::float8, NULL::date, g::numeric, g::float4, g%2=0, g%2=1 FROM generate_series(1,$1) g"
+        sql17Simple = "SELECT g, ('2000-01-01'::date + g::int4), ('2000-06-15'::date + g::int4), ('2000-01-01T00:00:00Z'::timestamptz + g * interval '1 second'), ('2020-06-15T12:00:00Z'::timestamptz + g * interval '1 minute'), 'row-' || g::text, 'item-' || g::text, g::float8 * 1.5, g::float8 * 2.5, (CASE WHEN g%2=1 THEN NULL ELSE g::int4 END), (CASE WHEN g%2=1 THEN NULL ELSE '' END)::text, (CASE WHEN g%2=1 THEN NULL ELSE 0.0 END)::float8, NULL::date, g::numeric, g::float4, g%2=0, g%2=1 FROM generate_series(1,?) g"
+        sql13 = "SELECT g, ('2000-01-01'::date + g::int4), ('2000-06-15'::date + g::int4), ('2000-01-01T00:00:00Z'::timestamptz + g * interval '1 second'), ('2020-06-15T12:00:00Z'::timestamptz + g * interval '1 minute'), 'row-' || g::text, 'item-' || g::text, g::float8 * 1.5, g::float8 * 2.5, (CASE WHEN g%2=1 THEN NULL ELSE g END)::int4, (CASE WHEN g%2=1 THEN NULL ELSE '' END)::text, (CASE WHEN g%2=1 THEN NULL ELSE 0.0 END)::float8, NULL::date FROM generate_series(1,$1) g"
+        sql13Simple = "SELECT g, ('2000-01-01'::date + g::int4), ('2000-06-15'::date + g::int4), ('2000-01-01T00:00:00Z'::timestamptz + g * interval '1 second'), ('2020-06-15T12:00:00Z'::timestamptz + g * interval '1 minute'), 'row-' || g::text, 'item-' || g::text, g::float8 * 1.5, g::float8 * 2.5, (CASE WHEN g%2=1 THEN NULL ELSE g END)::int4, (CASE WHEN g%2=1 THEN NULL ELSE '' END)::text, (CASE WHEN g%2=1 THEN NULL ELSE 0.0 END)::float8, NULL::date FROM generate_series(1,?) g"
     describe "Parsing 13-column rows into a List" $ do
       let hasqlListStmt =
             HasqlStmt.Statement
@@ -257,6 +266,16 @@ main = do
           bench ("hpgsql Record List (" ++ show n ++ " rows, Generically derived row decoder)") $
             withMultipleConnections numConcurrentConnections hpgsqlConnect Hpgsql.Connection.closeGracefully $ \conn -> do
               Hpgsql.queryWith (Hpgsql.rowDecoder @BenchRow) conn (Hpgsql.mkQuery sql17 (Hpgsql.Only n))
+      it ("hpgsql Record List (" ++ show n ++ " rows, fully inlined row decoder)") $
+        void $
+          bench ("hpgsql Record List (" ++ show n ++ " rows, fully inlined row decoder)") $
+            withMultipleConnections numConcurrentConnections hpgsqlConnect Hpgsql.Connection.closeGracefully $ \conn -> do
+              Hpgsql.queryWith fullyInlinedBenchRowDecoder conn (Hpgsql.mkQuery sql17 (Hpgsql.Only n))
+      it ("hpgsql Record List (" ++ show n ++ " rows, `singleField fieldDecoder` row decoder)") $
+        void $
+          bench ("hpgsql Record List (" ++ show n ++ " rows, `singleField fieldDecoder` row decoder)") $
+            withMultipleConnections numConcurrentConnections hpgsqlConnect Hpgsql.Connection.closeGracefully $ \conn -> do
+              Hpgsql.queryWith singleFieldBenchRowDecoder conn (Hpgsql.mkQuery sql17 (Hpgsql.Only n))
       it ("hasql Record List (" ++ show n ++ " rows)") $
         void $
           bench ("hasql Record List (" ++ show n ++ " rows)") $
@@ -301,11 +320,24 @@ main = do
               runResourceT @IO $ do
                 let res :: Stream (Of BenchRow) (ResourceT IO) () = StreamingPostgresSimple.query pgSimpleConn sql17Simple (PGSimple.Only n)
                 S.effects res
-      it ("postgresql-simple Record fold (" ++ show n ++ " rows, Generically derived row decoder)") $
+      it ("hpgsql Record Stream (" ++ show n ++ " rows, fully inlined row decoder)") $
+        void $
+          bench ("hpgsql Record Stream (" ++ show n ++ " rows, fully inlined row decoder)") $ do
+            withMultipleConnections numConcurrentConnections hpgsqlConnect Hpgsql.Connection.closeGracefully $ \conn -> do
+              res <- Hpgsql.querySWith fullyInlinedBenchRowDecoder conn (Hpgsql.mkQuery sql17 (Hpgsql.Only n))
+              S.effects res
+      it ("streaming-postgresql-simple Record Stream (" ++ show n ++ " rows)") $
+        void $
+          bench ("streaming-postgresql-simple Record Stream (" ++ show n ++ " rows)") $
+            withMultipleConnections numConcurrentConnections pgSimpleConnect PGSimple.close $ \pgSimpleConn -> do
+              runResourceT @IO $ do
+                let res :: Stream (Of BenchRow) (ResourceT IO) () = StreamingPostgresSimple.query pgSimpleConn sql17Simple (PGSimple.Only n)
+                S.effects res
+      it ("postgresql-simple Record fold (" ++ show n ++ " rows)") $
         void $
-          bench ("postgresql-simple Record fold (" ++ show n ++ " rows, Generically derived row decoder)") $
+          bench ("postgresql-simple Record fold (" ++ show n ++ " rows)") $
             withMultipleConnections numConcurrentConnections pgSimpleConnect PGSimple.close $ \pgSimpleConn -> do
-              PGSimple.fold pgSimpleConn sql17Simple (PGSimple.Only n) () (\() (!_ :: BenchRow) -> pure ())
+              PGSimple.fold pgSimpleConn "SELECT g, ('2000-01-01'::date + g::int4), ('2000-06-15'::date + g::int4), ('2000-01-01T00:00:00Z'::timestamptz + g * interval '1 second'), ('2020-06-15T12:00:00Z'::timestamptz + g * interval '1 minute'), 'row-' || g::text, 'item-' || g::text, g::float8 * 1.5, g::float8 * 2.5, NULL::int4, NULL::text, NULL::float8, NULL::date FROM generate_series(1,?) g" (PGSimple.Only n) () (\() (!_ :: BenchRow) -> pure ())
     describe "COPY FROM STDIN" $ do
       (conn, pgSimpleConn) <- runIO $ do
         hpgsqlConnInfo <- testConnInfo

hpgsql-simple-compat/src/Database/PostgreSQL/Simple/HpgsqlUtils.hs

@@ -104,9 +104,10 @@ toHpgsqlFieldDecoder fp =
     }
 
 fromHpgsqlFieldDecoder :: FieldDecoder a -> FieldParser a
-fromHpgsqlFieldDecoder dec = \f mbs -> Conversion $ \_encCtx -> case dec.fieldValueDecoder f mbs of
-  Right v -> Ok v
-  Left err -> Errors [toException $ userError $ show err]
+fromHpgsqlFieldDecoder dec = \f mbs -> Conversion $ \_encCtx ->
+  case dec.fieldValueDecoder f mbs of
+    Right v -> Ok v
+    Left err -> Errors [toException $ userError $ show err]
 
 -- | Given a Hpgsql query, returns the text format with question marks
 -- for query arguments and a row object. With both, you can call