Add SQLite backend

.github/workflows/ci.yml

@@ -103,11 +103,11 @@ jobs:
 
       - name: Make target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
-        run: mkdir -p modules/skunk/js/target modules/sql-core/.js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/skunk/jvm/target modules/core/.js/target modules/doobie-core/target modules/circe/.js/target modules/skunk/native/target modules/generic/.js/target modules/doobie-oracle/target modules/sql-core/.jvm/target modules/core/.jvm/target modules/sql-pg/native/target modules/doobie-mssql/target modules/sql-pg/js/target modules/circe/.native/target modules/generic/.native/target modules/sql-pg/jvm/target modules/sql-core/.native/target project/target
+        run: mkdir -p modules/skunk/js/target modules/sql-core/.js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/skunk/jvm/target modules/core/.js/target modules/doobie-core/target modules/circe/.js/target modules/skunk/native/target modules/generic/.js/target modules/doobie-oracle/target modules/sql-core/.jvm/target modules/core/.jvm/target modules/sql-pg/native/target modules/doobie-mssql/target modules/sql-pg/js/target modules/doobie-sqlite/target modules/circe/.native/target modules/generic/.native/target modules/sql-pg/jvm/target modules/sql-core/.native/target project/target
 
       - name: Compress target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
-        run: tar cf targets.tar modules/skunk/js/target modules/sql-core/.js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/skunk/jvm/target modules/core/.js/target modules/doobie-core/target modules/circe/.js/target modules/skunk/native/target modules/generic/.js/target modules/doobie-oracle/target modules/sql-core/.jvm/target modules/core/.jvm/target modules/sql-pg/native/target modules/doobie-mssql/target modules/sql-pg/js/target modules/circe/.native/target modules/generic/.native/target modules/sql-pg/jvm/target modules/sql-core/.native/target project/target
+        run: tar cf targets.tar modules/skunk/js/target modules/sql-core/.js/target modules/circe/.jvm/target modules/generic/.jvm/target modules/doobie-pg/target unidocs/target modules/core/.native/target modules/skunk/jvm/target modules/core/.js/target modules/doobie-core/target modules/circe/.js/target modules/skunk/native/target modules/generic/.js/target modules/doobie-oracle/target modules/sql-core/.jvm/target modules/core/.jvm/target modules/sql-pg/native/target modules/doobie-mssql/target modules/sql-pg/js/target modules/doobie-sqlite/target modules/circe/.native/target modules/generic/.native/target modules/sql-pg/jvm/target modules/sql-core/.native/target project/target
 
       - name: Upload target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')

build.sbt

@@ -24,6 +24,7 @@ val munitScalaCheckVersion = "1.3.0"
 val oracleDriverVersion = "23.26.2.0.0"
 val postgresVersion = "42.7.13"
 val skunkVersion = "1.0.0"
+val sqliteDriverVersion = "3.53.2.0"
 val shapeless2Version = "2.3.13"
 val shapeless3Version = "3.6.0"
 val sourcePosVersion = "1.2.0"
@@ -36,7 +37,7 @@ ThisBuild / scalaVersion := Scala2
 ThisBuild / crossScalaVersions := Seq(Scala2, Scala3)
 ThisBuild / tlJdkRelease := Some(11)
 
-ThisBuild / tlBaseVersion := "0.29"
+ThisBuild / tlBaseVersion := "0.30"
 ThisBuild / startYear := Some(2019)
 ThisBuild / licenses := Seq(License.Apache2)
 ThisBuild / developers := List(
@@ -171,6 +172,13 @@ lazy val nativeSettings = Seq(
   )
 )
 
+// Pin the forked test JVM to UTC: mssql-jdbc binds a zone-naive java.sql.Timestamp using the
+// ambient JVM zone, so MSSQL datetime tests fail off-UTC unless pinned. Only meaningful where
+// Test / fork := true.
+lazy val utcTestSettings = Seq(
+  Test / javaOptions += "-Duser.timezone=UTC"
+)
+
 lazy val modules: List[CompositeProject] = List(
   core,
   circe,
@@ -180,6 +188,7 @@ lazy val modules: List[CompositeProject] = List(
   doobiepg,
   doobieoracle,
   doobiemssql,
+  doobiesqlite,
   skunk,
   generic,
   docs,
@@ -272,6 +281,7 @@ lazy val doobiecore = project
   .disablePlugins(RevolverPlugin)
   .dependsOn(sqlcore.jvm % "test->test;compile->compile", circe.jvm)
   .settings(commonSettings)
+  .settings(utcTestSettings)
   .settings(
     name := "grackle-doobie-core",
     Test / fork := true,
@@ -291,6 +301,7 @@ lazy val doobiepg = project
     doobiecore % "test->test;compile->compile",
     sqlpg.jvm % "test->test;compile->compile")
   .settings(commonSettings)
+  .settings(utcTestSettings)
   .settings(
     name := "grackle-doobie-pg",
     Test / fork := true,
@@ -310,6 +321,7 @@ lazy val doobieoracle = project
   .disablePlugins(RevolverPlugin)
   .dependsOn(doobiecore % "test->test;compile->compile")
   .settings(commonSettings)
+  .settings(utcTestSettings)
   .settings(
     name := "grackle-doobie-oracle",
     Test / fork := true,
@@ -327,6 +339,7 @@ lazy val doobiemssql = project
   .disablePlugins(RevolverPlugin)
   .dependsOn(doobiecore % "test->test;compile->compile")
   .settings(commonSettings)
+  .settings(utcTestSettings)
   .settings(
     name := "grackle-doobie-mssql",
     Test / fork := true,
@@ -338,6 +351,38 @@ lazy val doobiemssql = project
     )
   )
 
+lazy val doobiesqlite = project
+  .in(file("modules/doobie-sqlite"))
+  .enablePlugins(AutomateHeaderPlugin)
+  .disablePlugins(RevolverPlugin)
+  .dependsOn(doobiecore % "test->test;compile->compile")
+  .settings(commonSettings)
+  .settings(utcTestSettings)
+  .settings(
+    name := "grackle-doobie-sqlite",
+    Test / fork := true,
+    Test / parallelExecution := false,
+    // SQLite has no docker service: unlike Oracle/MSSQL, whose containers auto-run the seed SQL
+    // mounted from testdata/<db>/, the test harness loads and executes testdata/sqlite/*.sql
+    // itself against a fresh temp database file per suite. Pass the directory as a system property
+    // (fork'd tests don't share the build's working directory) rather than relying on a relative
+    // path guess.
+    Test / javaOptions += s"-Dgrackle.sqlite.testdata=${(ThisBuild / baseDirectory).value / "testdata" / "sqlite"}",
+    // sqlite-jdbc's native cleanup on Connection#close touches JNI from what recent JDKs treat as
+    // a restricted context; without this the forked test JVM logs "restricted method" warnings and
+    // native handle teardown can throw spuriously. The flag only exists on JDK 17+ (JEP 412) -
+    // older JVMs, such as CI's temurin@11, refuse to start when given it (the forked JVM inherits
+    // the JDK sbt runs on), so it has to be supplied conditionally.
+    Test / javaOptions ++= {
+      if (sys.props("java.specification.version").toDouble >= 17)
+        Seq("--enable-native-access=ALL-UNNAMED")
+      else Nil
+    },
+    libraryDependencies ++= Seq(
+      "org.xerial" % "sqlite-jdbc" % sqliteDriverVersion
+    )
+  )
+
 lazy val skunk = crossProject(JVMPlatform, JSPlatform, NativePlatform)
   .crossType(CrossType.Full)
   .in(file("modules/skunk"))
@@ -354,6 +399,7 @@ lazy val skunk = crossProject(JVMPlatform, JSPlatform, NativePlatform)
       "org.typelevel" %% "log4cats-core" % log4catsVersion
     )
   )
+  .jvmSettings(utcTestSettings)
   .jvmSettings(
     Test / fork := true,
     Test / testOptions += Tests.Setup(_ =>
@@ -486,6 +532,7 @@ lazy val unidocs = project
       doobiepg,
       doobieoracle,
       doobiemssql,
+      doobiesqlite,
       skunk.jvm,
       generic.jvm
     )

modules/doobie-core/src/test/scala/DoobieDatabaseSuite.scala

@@ -18,13 +18,12 @@ package grackle.doobie.test
 import java.time.Duration
 
 import io.circe.{Decoder => CDecoder, Encoder => CEncoder}
-import munit.CatsEffectSuite
 import org.typelevel.doobie.Meta
 
 import grackle.doobie.DoobieMappingLike
 import grackle.sql.test._
 
-trait DoobieDatabaseSuite extends CatsEffectSuite {
+trait DoobieDatabaseSuite extends SqlDatabaseSuite {
   trait DoobieTestMapping[F[_]] extends DoobieMappingLike[F] with SqlTestMapping[F] {
 
     type TestCodec[T] = (Meta[T], Boolean)

modules/doobie-mssql/src/main/scala/DoobieMSSqlMapping.scala

@@ -90,13 +90,18 @@ trait DoobieMSSqlMappingLike[F[_]] extends DoobieMappingLike[F] with SqlMappingL
     if (s.orders.isEmpty) s
     else s.toSubquery(s.table.name + "_encaps", Laterality.NotLateral)
 
+  def unionBranchToFragment(branch: Fragment): Fragment = Fragments.parentheses(branch)
+
   def mkLateral(inner: Boolean): Laterality =
     Laterality.Apply(inner)
 
-  def defaultOffsetForSubquery(subquery: SqlQuery): SqlQuery =
-    subquery match {
+  // MSSQL's grammar requires an ORDER BY inside a derived table to be paired with an
+  // OFFSET/FETCH clause; at the query root the pairing is optional and OFFSET 0 ROWS is a
+  // harmless no-op, so the default can be supplied unconditionally.
+  def normalizeOffsetLimit(query: SqlQuery): SqlQuery =
+    query match {
       case s: SqlSelect if s.orders.nonEmpty && s.offset.isEmpty => s.copy(offset = 0.some)
-      case _ => subquery
+      case _ => query
     }
 
   def defaultOffsetForLimit(limit: Option[Int]): Option[Int] =

modules/doobie-oracle/src/main/scala/DoobieOracleMapping.scala

@@ -89,8 +89,9 @@ trait DoobieOracleMappingLike[F[_]] extends DoobieMappingLike[F] with SqlMapping
     ) // TODO: check that passing orders works with Oracle
 
   def encapsulateUnionBranch(s: SqlSelect): SqlSelect = s
+  def unionBranchToFragment(branch: Fragment): Fragment = Fragments.parentheses(branch)
   def mkLateral(inner: Boolean): Laterality = Laterality.Lateral
-  def defaultOffsetForSubquery(subquery: SqlQuery): SqlQuery = subquery
+  def normalizeOffsetLimit(query: SqlQuery): SqlQuery = query
   def defaultOffsetForLimit(limit: Option[Int]): Option[Int] = None
 
   def orderToFragment(col: Fragment, ascending: Boolean, nullsLast: Boolean): Fragment = {

modules/doobie-sqlite/src/main/scala/DoobieSqliteMapping.scala

@@ -0,0 +1,175 @@
+// Copyright (c) 2016-2025 Association of Universities for Research in Astronomy, Inc. (AURA)
+// Copyright (c) 2016-2025 Grackle Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package grackle.doobie.sqlite
+
+import cats.effect.Sync
+import cats.syntax.all._
+import org.typelevel.doobie.Transactor
+
+import grackle.Mapping
+import grackle.Query.OrderSelection
+import grackle.doobie._
+import grackle.sql._
+
+abstract class DoobieSqliteMapping[F[_]](
+    val transactor: Transactor[F],
+    val monitor: DoobieMonitor[F]
+)(
+    implicit val M: Sync[F]
+) extends Mapping[F]
+    with DoobieSqliteMappingLike[F]
+
+/**
+ * SQLite lacks two SQL constructs the shared query builder in `grackle.sql.SqlMappingLike`
+ * (`modules/sql-core`) otherwise assumes are always available; each is bridged by a dialect
+ * hook that the other backends implement with their previous behavior:
+ *
+ *   - '''No correlated FROM-clause subqueries.''' SQLite has no `LATERAL` keyword and no other
+ *     way for a subquery in the FROM clause to reference a sibling table's columns, so
+ *     `mkLateral` below answers `NotLateral` - the only possible rendering - and
+ *     `SqlMappingLike` derives `supportsLateralJoin = false` from that. Most queries that ask
+ *     for lateral evaluation don't actually need it: the correlation is supplied independently
+ *     by the `JOIN ... ON` clause `SqlSelect.nest` builds regardless of dialect. See
+ *     `supportsLateralJoin`'s doc comment for the consequences (an omitted redundant predicate,
+ *     gated "Case 1" fast paths) and the performance trade-off.
+ *   - '''No parenthesized UNION branches.''' SQLite's compound-select grammar is
+ *     `select-core (compound-operator select-core)*` - a branch can never be parenthesized,
+ *     unconditionally, so `unionBranchToFragment` below renders branches bare. A branch
+ *     carrying its own order, offset, or limit can't be expressed inline either and is wrapped
+ *     in a derived-table subquery by `encapsulateUnionBranch`, which extends the MSSQL
+ *     treatment (orders only) to offset and limit as well.
+ */
+trait DoobieSqliteMappingLike[F[_]] extends DoobieMappingLike[F] with SqlMappingLike[F] {
+  import SqlQuery.SqlSelect
+  import TableExpr.Laterality
+
+  def collateToFragment: Fragment =
+    Fragments.const(" COLLATE BINARY")
+
+  def aliasDefToFragment(alias: String): Fragment =
+    Fragments.const(s" AS $alias")
+
+  // SQLite's LIMIT/OFFSET clause is anchored on the `LIMIT` keyword: `OFFSET` (or a comma) is
+  // only legal *inside* a LIMIT clause, never as a standalone top-level clause, and never before
+  // the word LIMIT. That's incompatible with the fixed `offsetToFragment |+| limitToFragment`
+  // rendering order used by the shared query builder (which suits Postgres, where either order is
+  // legal, and MSSQL/Oracle's OFFSET-anchored `OFFSET .. FETCH ..`). We route around this by using
+  // SQLite's legacy MySQL-style comma form `LIMIT <offset>, <limit>`: offsetToFragment opens the
+  // clause and limitToFragment supplies the trailing operand. The two companion hooks below
+  // guarantee the pair is always complete: defaultOffsetForLimit supplies offset 0 whenever a
+  // limit is present, and normalizeOffsetLimit supplies `LIMIT -1` (SQLite's documented "no upper
+  // bound" idiom) whenever an explicit offset has no limit to pair with.
+  def offsetToFragment(offset: Fragment): Fragment =
+    Fragments.const(" LIMIT ") |+| offset |+| Fragments.const(", ")
+
+  def limitToFragment(limit: Fragment): Fragment =
+    limit
+
+  // SQLite's LIKE is ASCII case-insensitive by default and has no ILIKE, so genuinely
+  // case-sensitive matching requires the connection-level `PRAGMA case_sensitive_like = ON`
+  // (there's no per-expression equivalent - callers building a Transactor for this mapping need
+  // to set that pragma, e.g. via SQLiteConfig; see DoobieSqliteDatabaseSuite for a worked
+  // example). Since that pragma is global to the connection, not per-query, we can't just fall
+  // back to a bare LIKE for the case-insensitive branch once it's enabled - both branches need to
+  // be made explicit, exactly as Oracle/MSSQL do: normalise to upper case for case-insensitive
+  // matches (which is then case-insensitive regardless of the pragma), and compare as-is
+  // (case-sensitive, relying on the pragma) otherwise.
+  def likeToFragment(expr: Fragment, pattern: String, caseInsensitive: Boolean): Fragment = {
+    val casedExpr =
+      if (caseInsensitive) Fragments.const("UPPER(") |+| expr |+| Fragments.const(s")")
+      else expr
+    val casedPattern = if (caseInsensitive) pattern.toUpperCase else pattern
+    casedExpr |+| Fragments.const(s" LIKE ") |+| Fragments.bind(stringEncoder, casedPattern)
+  }
+
+  // SQLite is dynamically typed, and its CAST accepts arbitrary type names (falling back to a
+  // best-guess type affinity for anything it doesn't recognise), so a typed NULL can just reuse
+  // whatever name the driver reports - no per-type remapping needed, unlike Oracle/MSSQL.
+  def ascribedNullToFragment(codec: Codec): Fragment =
+    Fragments.sqlTypeName(codec) match {
+      case Some(name) => Fragments.const(s"CAST(NULL AS $name)")
+      case None => Fragments.const("NULL")
+    }
+
+  def collateSelected: Boolean = false
+
+  def distinctOnToFragment(dcols: List[Fragment]): Fragment =
+    Fragments.const("DISTINCT ")
+
+  def distinctOrderColumn(
+      owner: ColumnOwner,
+      col: SqlColumn,
+      predCols: List[SqlColumn],
+      orders: List[OrderSelection[_]]): SqlColumn =
+    SqlColumn.FirstValueColumn(owner, col, predCols, orders)
+
+  // A compound SELECT (UNION ALL/etc.) may only have a single ORDER BY/LIMIT/OFFSET, trailing the
+  // whole compound statement - an individual branch can't carry its own, parenthesized or not.
+  // Branches that do (grackle pushes a per-branch limit into paged-wrapper "items" branches, for
+  // example) must be wrapped in a derived table instead, extending the MSSQL treatment of orders
+  // to offset and limit as well.
+  def encapsulateUnionBranch(s: SqlSelect): SqlSelect =
+    if (s.orders.isEmpty && s.offset.isEmpty && s.limit.isEmpty) s
+    else s.toSubquery(s.table.name + "_encaps", Laterality.NotLateral)
+
+  // A branch of a compound select can never be parenthesized, not even a "plain" one with no
+  // order/limit, so branches render bare. See unionBranchToFragment's doc comment on
+  // SqlMappingLike for why dropping the parens is safe.
+  def unionBranchToFragment(branch: Fragment): Fragment = branch
+
+  // SQLite has no LATERAL/APPLY mechanism at all, so NotLateral (plain subquery, no keyword) is
+  // the only possible answer; SqlMappingLike derives supportsLateralJoin = false from it, which
+  // omits the parent-constraint predicate only a lateral subquery could resolve and gates the
+  // "Case 1" fast paths - see that member's doc comment.
+  def mkLateral(inner: Boolean): Laterality =
+    Laterality.NotLateral
+
+  // Mirror image of defaultOffsetForLimit, but at the query-tree level: a select with an
+  // explicit offset but no limit gets SQLite's documented idiom for "no upper bound",
+  // `LIMIT -1`, so the comma-form OFFSET/LIMIT pairing in offsetToFragment always has a second
+  // operand to pair with.
+  def normalizeOffsetLimit(query: SqlQuery): SqlQuery =
+    query match {
+      case s: SqlSelect if s.offset.nonEmpty && s.limit.isEmpty => s.copy(limit = (-1).some)
+      case _ => query
+    }
+
+  // See offsetToFragment: forcing a default offset of 0 whenever a limit is present guarantees
+  // the comma-form clause is always rendered as a matched `LIMIT offset, limit` pair.
+  def defaultOffsetForLimit(limit: Option[Int]): Option[Int] =
+    limit.as(0)
+
+  // Modern SQLite (>= 3.30) supports NULLS FIRST/LAST natively, but unlike Postgres/Oracle its
+  // default places NULLs low (first in ASC, last in DESC), so the explicit clause is needed on
+  // the mirror-image cases relative to the pg dialect - the same polarity correction the MSSQL
+  // dialect makes. Pinned by NullOrderingSuite.
+  def orderToFragment(col: Fragment, ascending: Boolean, nullsLast: Boolean): Fragment = {
+    val dir = if (ascending) Fragments.empty else Fragments.const(" DESC")
+    val nulls =
+      if (nullsLast && ascending)
+        Fragments.const(" NULLS LAST ")
+      else if (!nullsLast && !ascending)
+        Fragments.const(" NULLS FIRST ")
+      else
+        Fragments.empty
+
+    col |+| dir |+| nulls
+  }
+
+  // SQLite sorts NULL as lower than any non-NULL value by default (NULLs first in ASC), the same
+  // convention as MSSQL.
+  def nullsHigh: Boolean = false
+}