diff --git a/.github/doltlite-patches/canonicalize-schema-sql-comments.patch b/.github/doltlite-patches/canonicalize-schema-sql-comments.patch new file mode 100644 index 000000000..c2a5d9948 --- /dev/null +++ b/.github/doltlite-patches/canonicalize-schema-sql-comments.patch @@ -0,0 +1,47 @@ +Strip SQL comments while canonicalizing sqlite_schema SQL + +Doltlite canonicalizes schema SQL by collapsing whitespace before storing +sqlite_schema rows after rollback and restore paths. When the SQL contains +line comments, collapsing newlines while preserving `--` makes the comment +consume the rest of the CREATE statement. SQLite then reports a malformed +schema with incomplete input. + +Skip line and block comments outside quoted strings during schema SQL +canonicalization. This keeps canonicalized schema text parseable while +preserving the existing whitespace normalization behavior. + +diff --git a/src/doltlite_hashof.c b/src/doltlite_hashof.c +index 1563b18..7f8e073 100644 +--- a/src/doltlite_hashof.c ++++ b/src/doltlite_hashof.c +@@ -223,6 +223,30 @@ char *doltliteCanonicalizeSchemaSql(const char *zSql, const char *zName){ + if( c=='"' ) inDouble = 0; + continue; + } ++ if( c=='-' && z[1]=='-' ){ ++ z += 2; ++ while( *z && *z!='\n' && *z!='\r' ) z++; ++ pendingSpace = 1; ++ if( *z==0 ){ ++ z--; ++ }else if( z[0]=='\r' && z[1]=='\n' ){ ++ z++; ++ } ++ continue; ++ } ++ if( c=='/' && z[1]=='*' ){ ++ const char *zEnd = z + 2; ++ while( zEnd[0] && !(zEnd[0]=='*' && zEnd[1]=='/') ){ ++ zEnd++; ++ } ++ pendingSpace = 1; ++ if( zEnd[0]=='*' ){ ++ z = zEnd + 1; ++ }else{ ++ z = zEnd - 1; ++ } ++ continue; ++ } + if( c=='\'' ){ + if( pendingSpace && sqlite3_str_length(pStr)>0 ){ + sqlite3_str_appendchar(pStr, 1, ' '); diff --git a/.github/doltlite-patches/configurable-journal-mode.patch b/.github/doltlite-patches/configurable-journal-mode.patch new file mode 100644 index 000000000..1ad3e8f61 --- /dev/null +++ b/.github/doltlite-patches/configurable-journal-mode.patch @@ -0,0 +1,70 @@ +Emulate configurable journal modes for the compatibility suite + +Doltlite's chunk store has no rollback-journal or WAL sidecars, so upstream +reports a fixed WAL mode and ignores PRAGMA journal_mode changes. The +PDO-compatible mysql-on-sqlite API exposes journal mode configuration and +follows stock SQLite behavior when WAL cannot be opened. + +Use the existing pager-shim journalMode field as logical compatibility state. +Start file databases in SQLite's default DELETE mode, allow the normal VDBE +journal-mode transitions, and reject a WAL transition when the database +directory is not writable. The storage engine remains unchanged. + +diff --git a/src/pager_shim.c b/src/pager_shim.c +index d41e656..c362474 100644 +--- a/src/pager_shim.c ++++ b/src/pager_shim.c +@@ -554,7 +554,7 @@ PagerShim *pagerShimCreate( + || strcmp(pShim->zFilename, ":memory:")==0 ){ + pShim->journalMode = PAGER_JOURNALMODE_MEMORY; + }else{ +- pShim->journalMode = PAGER_JOURNALMODE_WAL; ++ pShim->journalMode = PAGER_JOURNALMODE_DELETE; + } + pShim->eLock = 0; + pShim->iDataVersion = 1; +diff --git a/src/prolly_btree.c b/src/prolly_btree.c +index 9cd8b18..8b1130b 100644 +--- a/src/prolly_btree.c ++++ b/src/prolly_btree.c +@@ -9959,4 +9959,24 @@ int doltliteCheckRepoGraphIntegrity(Btree *p, int mxErr, int *pnErr){ + int sqlite3BtreeSetVersion(Btree *p, int iVersion){ ++ if( iVersion==2 && p->pBt->pPagerShim ){ ++ PagerShim *pShim = p->pBt->pPagerShim; ++ const char *zSlash = strrchr(pShim->zFilename, '/'); ++ char *zDir; ++ int isWritable = 0; ++ int rc; ++ if( zSlash ){ ++ int nDir = (int)(zSlash - pShim->zFilename); ++ if( nDir==0 ) nDir = 1; ++ zDir = sqlite3_mprintf("%.*s", nDir, pShim->zFilename); ++ }else{ ++ zDir = sqlite3_mprintf("."); ++ } ++ if( !zDir ) return SQLITE_NOMEM; ++ rc = sqlite3OsAccess(pShim->pVfs, zDir, ++ SQLITE_ACCESS_READWRITE, &isWritable); ++ sqlite3_free(zDir); ++ if( rc!=SQLITE_OK ) return rc; ++ if( !isWritable ) return SQLITE_CANTOPEN; ++ } + if( p->inTrans!=TRANS_WRITE ){ + int rc = sqlite3BtreeBeginTrans(p, 2, 0); + if( rc!=SQLITE_OK ) return rc; +diff --git a/src/vdbe.c b/src/vdbe.c +index fae65b0..2ec8d06 100644 +--- a/src/vdbe.c ++++ b/src/vdbe.c +@@ -8514,11 +8514,0 @@ case OP_JournalMode: { /* out2 */ +-#ifdef DOLTLITE_PROLLY +- if( eNew!=eOld && sqlite3BtreeIsDoltliteFormat(pBt) ){ +- /* doltlite keeps data in a content-addressed chunk store with no +- ** rollback-journal or WAL sidecar, so journal_mode is inapplicable. +- ** Follow SQLite's convention of silently ignoring an inapplicable +- ** journal_mode request (as auto_vacuum already is) rather than erroring: +- ** keep the fixed mode and report it back. */ +- eNew = eOld; +- } +-#endif +- diff --git a/.github/doltlite-patches/keep-driver-rowid-tables.patch b/.github/doltlite-patches/keep-driver-rowid-tables.patch new file mode 100644 index 000000000..741d529d4 --- /dev/null +++ b/.github/doltlite-patches/keep-driver-rowid-tables.patch @@ -0,0 +1,17 @@ +Keep mysql-on-sqlite tables as rowid tables + +Doltlite auto-converts ordinary primary-key tables without an integer rowid +to WITHOUT ROWID. mysql-on-sqlite currently relies on rowid access for +table recreation, limited UPDATE/DELETE rewrites, on-update triggers, +zero-column result statements, and information-schema ordering. Keep the +tables in rowid mode for the Doltlite compatibility test run. + +--- a/src/build.c ++++ b/src/build.c +@@ -2794,5 +2794,5 @@ void sqlite3EndTable( + && (p->tabFlags & TF_HasPrimaryKey)!=0 + && p->iPKey<0 + && (tabOpts & TF_WithoutRowid)==0 ){ +- tabOpts |= TF_WithoutRowid; ++ /* patched off in CI: keep mysql-on-sqlite tables as rowid tables */ (void)0; + } diff --git a/.github/workflows/phpunit-tests-doltlite.yml b/.github/workflows/phpunit-tests-doltlite.yml new file mode 100644 index 000000000..617a067ac --- /dev/null +++ b/.github/workflows/phpunit-tests-doltlite.yml @@ -0,0 +1,131 @@ +name: PHPUnit Tests (Doltlite) + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + test: + name: PHP ${{ matrix.php }} / Doltlite + runs-on: ubuntu-latest + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + php: [ '8.3' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Doltlite build dependencies + run: sudo apt-get update && sudo apt-get install -y build-essential zlib1g-dev tcl-dev + + - name: Check out Doltlite + uses: actions/checkout@v4 + with: + repository: dolthub/doltlite + ref: 0745bbab29778393780b220d2a91fc5fa753853f + path: doltlite-src + + - name: Patch Doltlite for mysql-on-sqlite compatibility + # Keep driver tables as rowid tables, preserve schema SQL comments + # safely across rollback, and expose SQLite-compatible logical journal + # modes through Doltlite's pager shim. + run: | + patch -d doltlite-src -p1 --no-backup-if-mismatch \ + < .github/doltlite-patches/keep-driver-rowid-tables.patch + patch -d doltlite-src -p1 --no-backup-if-mismatch \ + < .github/doltlite-patches/canonicalize-schema-sql-comments.patch + patch -d doltlite-src -p1 --no-backup-if-mismatch \ + < .github/doltlite-patches/configurable-journal-mode.patch + + - name: Build Doltlite shared library + run: | + mkdir -p build + cd build + # Same feature set we enable for stock-SQLite runs. PHP's pdo_sqlite + # pulls in sqlite3_column_table_name, which needs COLUMN_METADATA. + CFLAGS="-DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_FTS5 -DSQLITE_USE_URI -DSQLITE_ENABLE_JSON1" \ + ../configure + make -j"$(nproc)" doltlite-lib + ls -la libdoltlite* sqlite3.h + working-directory: doltlite-src + + - name: Set up PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '${{ matrix.php }}' + tools: phpunit-polyfills + + - name: Install Doltlite as the system libsqlite3 + # Replace the system libsqlite3 the dynamic linker resolves for + # pdo_sqlite with the Doltlite build. This is more reliable than + # LD_PRELOAD, which loses to pdo_sqlite's DT_NEEDED on libsqlite3.so.0. + run: | + SRC="${GITHUB_WORKSPACE}/doltlite-src/build/libdoltlite.so" + test -f "$SRC" + + # Drop the build into /usr/local/lib and update the linker cache before + # replacing the system libsqlite3 soname. Running ldconfig after the + # replacement can restore Ubuntu's stock libsqlite3.so.0 symlink. + sudo install -m 0755 "$SRC" /usr/local/lib/libdoltlite.so + sudo ln -sf /usr/local/lib/libdoltlite.so /usr/local/lib/libdoltlite.so.0 + + sudo ldconfig + + sudo ln -sf /usr/local/lib/libdoltlite.so /usr/local/lib/libsqlite3.so + sudo ln -sf /usr/local/lib/libdoltlite.so /usr/local/lib/libsqlite3.so.0 + + MULTIARCH_DIR="$(dirname "$(readlink -f /lib/x86_64-linux-gnu/libsqlite3.so.0)")" + sudo rm -f "${MULTIARCH_DIR}/libsqlite3.so.0" + sudo ln -s /usr/local/lib/libdoltlite.so "${MULTIARCH_DIR}/libsqlite3.so.0" + + echo "libsqlite3.so.0 target:" + readlink -f /lib/x86_64-linux-gnu/libsqlite3.so.0 + + echo "ldd pdo_sqlite:" + ldd "$(php -r 'echo ini_get("extension_dir");')/pdo_sqlite.so" + + - name: Verify Doltlite is active and the rowid patch is applied + run: | + VERSION=$(php -r 'echo (new PDO("sqlite::memory:"))->query("SELECT SQLITE_VERSION()")->fetch()[0];') + ENGINE=$(php -r 'echo (new PDO("sqlite::memory:"))->query("SELECT doltlite_engine()")->fetch()[0];') + echo "SQLITE_VERSION() = ${VERSION}" + echo "doltlite_engine() = ${ENGINE}" + if [ "${ENGINE}" != "prolly" ]; then + echo "::error::Doltlite is not active (expected doltlite_engine() = 'prolly', got '${ENGINE}')" + exit 1 + fi + STATS_ROWID=$(php -r ' + $db = new PDO("sqlite::memory:"); + $db->exec("CREATE TABLE _wp_sqlite_mysql_information_schema_statistics (table_schema TEXT, table_name TEXT, index_name TEXT, seq_in_index INT, PRIMARY KEY (table_schema, table_name, index_name, seq_in_index)) STRICT"); + $db->exec("INSERT INTO _wp_sqlite_mysql_information_schema_statistics VALUES (\"s\", \"t\", \"i\", 1)"); + echo $db->query("SELECT rowid FROM _wp_sqlite_mysql_information_schema_statistics")->fetch()[0]; + ' 2>&1) + echo "statistics metadata rowid = ${STATS_ROWID}" + if ! [[ "${STATS_ROWID}" =~ ^[0-9]+$ ]]; then + echo "::error::rowid-preservation patch did not take effect: ${STATS_ROWID}" + exit 1 + fi + + - name: Install Composer dependencies (root) + uses: ramsey/composer-install@v3 + with: + ignore-cache: "yes" + composer-options: "--optimize-autoloader" + + - name: Install Composer dependencies (mysql-on-sqlite) + uses: ramsey/composer-install@v3 + with: + working-directory: packages/mysql-on-sqlite + ignore-cache: "yes" + composer-options: "--optimize-autoloader" + + - name: Run PHPUnit tests against Doltlite + run: php ./vendor/bin/phpunit -c ./phpunit.xml.dist + working-directory: packages/mysql-on-sqlite diff --git a/packages/mysql-on-sqlite/tests/WP_PDO_MySQL_On_SQLite_PDO_API_Tests.php b/packages/mysql-on-sqlite/tests/WP_PDO_MySQL_On_SQLite_PDO_API_Tests.php index 404481a34..3c4d42054 100644 --- a/packages/mysql-on-sqlite/tests/WP_PDO_MySQL_On_SQLite_PDO_API_Tests.php +++ b/packages/mysql-on-sqlite/tests/WP_PDO_MySQL_On_SQLite_PDO_API_Tests.php @@ -598,7 +598,13 @@ public function data_pdo_fetch_methods(): Generator { } private function remove_database_files( string $path ): void { - foreach ( array( $path, $path . '-wal', $path . '-shm', $path . '-journal' ) as $file ) { + foreach ( array( + $path, + $path . '-wal', + $path . '-shm', + $path . '-journal', + dirname( $path ) . '/.' . basename( $path ) . '-lock', + ) as $file ) { if ( file_exists( $file ) ) { unlink( $file ); } diff --git a/packages/mysql-on-sqlite/tests/WP_SQLite_Connection_Tests.php b/packages/mysql-on-sqlite/tests/WP_SQLite_Connection_Tests.php index f598f94ec..1995a7d1e 100644 --- a/packages/mysql-on-sqlite/tests/WP_SQLite_Connection_Tests.php +++ b/packages/mysql-on-sqlite/tests/WP_SQLite_Connection_Tests.php @@ -34,6 +34,7 @@ public function tearDown(): void { $this->db_path . '-wal', $this->db_path . '-shm', $this->db_path . '-journal', + dirname( $this->db_path ) . '/.' . basename( $this->db_path ) . '-lock', ) as $path ) { if ( is_string( $path ) && file_exists( $path ) ) { unlink( $path ); diff --git a/packages/mysql-on-sqlite/tests/WP_SQLite_Driver_Concurrency_Tests.php b/packages/mysql-on-sqlite/tests/WP_SQLite_Driver_Concurrency_Tests.php index 136956044..3aa398983 100644 --- a/packages/mysql-on-sqlite/tests/WP_SQLite_Driver_Concurrency_Tests.php +++ b/packages/mysql-on-sqlite/tests/WP_SQLite_Driver_Concurrency_Tests.php @@ -24,6 +24,7 @@ public function tearDown(): void { $this->db_path . '-wal', $this->db_path . '-shm', $this->db_path . '-journal', + dirname( $this->db_path ) . '/.' . basename( $this->db_path ) . '-lock', ) as $path ) { if ( is_string( $path ) && file_exists( $path ) ) { unlink( $path );