diff --git a/CHANGELOG.md b/CHANGELOG.md index 81ac1da..67251fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `cacheLockTimeout` to configure the raster cache lock wait independently of upstream HTTP timeouts. +- `CachePruner::prune()` for scheduled raster tile retention, including obsolete namespaces and unused metadata. + +### Fixed + +- Close cache lock handles on successful and failed requests, and retry when a cleaner replaces the metadata inode or removes an empty cache directory. + ## [2.4.1] - 2026-08-28 ### Fixed diff --git a/README.md b/README.md index 69bd3ae..f109eae 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ sending output. The configuration defines the behavior of the proxy. * `cacheServerPath`: Base directory for caching tiles and map assets. +* `cacheLockTimeout`: (Raster tiles, optional) Maximum seconds to wait for a tile's exclusive cache lock. + Defaults to `2.85` for compatibility; set it above the full upstream fetch and processing budget. * `ops`: (Raster tiles) Operation pipeline for bitmap tile requests. * `mapAssetBasePath`: (Mapbox styles) URL path prefix for proxied style JSON, vector tiles, sprites, and glyphs. * `styles`: (Mapbox styles) Named style configurations for `MapboxStyleProxy`. @@ -135,6 +137,42 @@ requests. HTTP(S) fetches go through Guzzle; `file://` URLs are read from disk. For tile pipelines, set `upstreamHttp` at the root of the config. A `src` operation can override it with its own `upstreamHttp` block. +### Raster cache retention + +`cacheServerTtl` controls freshness: an expired tile is refreshed on its next request, and a failed refresh can +still serve its cached copy. It does not evict unused files. + +Call `CachePruner::prune()` from cron or your application's scheduler to remove raster tile files older than a +separate retention period. This includes obsolete `cacheServerName` namespaces, old metadata without retained +tiles, and empty directories. The return value counts deleted files. For example, run this daily: + +```php +use OpenMapsight\TileProxy\CachePruner; + +$deleted = CachePruner::prune('/var/cache/mapsight-tile-proxy', 30 * 86400); +``` + +Choose a retention period longer than your raster `cacheServerTtl` values and any upstream minimum caching +requirements. Age is measured from tile writes, not filesystem atime; metadata tracks the last completed request. +Retention limits the age of stored tiles, not total disk usage. The reserved `mapbox-style-proxy` subtree, including +its empty directories, is excluded from raster cleanup. + +The pruner takes the same per-tile lock as `Base`, skips busy tiles without waiting, and rechecks timestamps under +the lock. Waiting requests reopen metadata that was removed by the pruner before using the cache. Run cleanup +only after all processes sharing the cache use this version's locking, and leave `yoloOnLockTimeout` disabled. +Use a POSIX filesystem supporting `flock()` across those processes. Cache directories are reserved for proxy files; +symlinks are not followed by the pruner. + +For an upstream timeout of ten seconds plus tile processing, set `cacheLockTimeout` to a larger budget, for example: + +```jsonc +"cacheLockTimeout": 15, +"upstreamHttp": { "timeout": 10 } +``` + +Zero means a single non-blocking lock attempt. Negative or non-finite timeouts are rejected. Locks are closed on +both successful requests and failures; the default remains 2.85 seconds for existing deployments. + ### Error logging Most failures are handled gracefully in HTTP responses, but are otherwise silent unless logging is enabled. diff --git a/classes/Base.php b/classes/Base.php index 67489f2..16d09d3 100644 --- a/classes/Base.php +++ b/classes/Base.php @@ -45,23 +45,13 @@ public static function handleTileRequest( $cachePath = static::getCachePath($cfg, $reqArgs); $metadataPath = $cachePath . '-.metadata'; - Utils::mkdirp(dirname($cachePath)); - $lockH = fopen($metadataPath, 'cb'); - for ($lockTry = 1; ; ++$lockTry) { - if (flock($lockH, LOCK_EX | LOCK_NB) === true) { - break; - } - - if (10 <= $lockTry) { - if (isset($cfg['yoloOnLockTimeout']) && $cfg['yoloOnLockTimeout'] === true) { - @error_log('Yoloed lock for "' . $metadataPath . '"' . "\n"); - break; - } - + $lock = FileLock::acquire($metadataPath, (float)($cfg['cacheLockTimeout'] ?? 2.85)); + if ($lock === null) { + if (isset($cfg['yoloOnLockTimeout']) && $cfg['yoloOnLockTimeout'] === true) { + @error_log('Yoloed lock for "' . $metadataPath . '"' . "\n"); + } else { throw new RuntimeException('Can not lock "' . $metadataPath . '"'); } - - usleep($lockTry * $lockTry * 10 * 1000); } try { @@ -83,10 +73,13 @@ public static function handleTileRequest( $res->checkpointCache(); Metadata::save($meta); + if (!touch($metadataPath)) { + throw new RuntimeException('Could not update cache access time "' . $metadataPath . '"'); + } return static::buildTileResponse($res); } finally { - flock($lockH, LOCK_UN); + $lock?->release(); } } diff --git a/classes/CachePruner.php b/classes/CachePruner.php new file mode 100644 index 0000000..eea2e80 --- /dev/null +++ b/classes/CachePruner.php @@ -0,0 +1,184 @@ + new RecursiveDirectoryIterator( + $cacheServerPath, RecursiveDirectoryIterator::SKIP_DOTS + ) + ); + if ($directories === null) { + return 0; + } + + $files = new RecursiveIteratorIterator( + new RecursiveCallbackFilterIterator( + $directories, + static fn (SplFileInfo $file): bool => $file->getPathname() !== $mapboxCachePath + ), + RecursiveIteratorIterator::CHILD_FIRST, + RecursiveIteratorIterator::CATCH_GET_CHILD + ); + + foreach ($files as $file) { + if ($file->isLink()) { + continue; + } + + if ($file->isDir()) { + $deleted += self::pruneDirectory($file->getPathname(), $cutoff); + // Non-empty directories and directories in use are left in place. + @rmdir($file->getPathname()); + } + } + + $deleted += self::pruneDirectory($cacheServerPath, $cutoff); + return $deleted; + } + + private static function pruneDirectory(string $path, int $cutoff): int + { + $files = self::openDirectory($path, static fn (): DirectoryIterator => new DirectoryIterator($path)); + if ($files === null) { + return 0; + } + + $groups = []; + foreach ($files as $file) { + if ($file->isLink() || !$file->isFile()) { + continue; + } + + if (preg_match('/^(\d+)-\.metadata$/D', $file->getFilename(), $matches) === 1) { + $groups[$matches[1]]['metadata'] = $file->getPathname(); + } elseif (preg_match('/^(\d+)-.*-\d+$/D', $file->getFilename(), $matches) === 1) { + $groups[$matches[1]]['tiles'][] = $file->getPathname(); + } + } + + $deleted = 0; + foreach ($groups as $group) { + if (isset($group['metadata'])) { + $deleted += self::pruneTile($group['metadata'], $group['tiles'] ?? [], $cutoff); + } + } + + return $deleted; + } + + /** + * @template T of DirectoryIterator + * @param callable(): T $open + * @return T|null + */ + private static function openDirectory(string $path, callable $open): ?DirectoryIterator + { + for ($attempt = 0; ; ++$attempt) { + try { + return $open(); + } catch (UnexpectedValueException $error) { + clearstatcache(true, $path); + if (!is_dir($path)) { + return null; + } + // A request may have recreated the directory after the failed open. Retry once. + if ($attempt > 0) { + throw $error; + } + } + } + } + + /** @param list $tiles */ + private static function pruneTile(string $metadataPath, array $tiles, int $cutoff): int + { + $lock = FileLock::acquire($metadataPath, 0, false); + if ($lock === null) { + return 0; + } + + try { + $deleted = 0; + $hasTiles = false; + + foreach ($tiles as $tilePath) { + if (self::isExpired($tilePath, $cutoff)) { + self::delete($tilePath); + ++$deleted; + } else { + $hasTiles = true; + } + } + + // A failed request can checkpoint a new namespace after the scan without touching + // metadata. Recheck the directory under the lock before removing its lock file. + if (!$hasTiles && self::isExpired($metadataPath, $cutoff) && !self::hasTiles($metadataPath)) { + self::delete($metadataPath); + ++$deleted; + } + + return $deleted; + } finally { + $lock->release(); + } + } + + private static function hasTiles(string $metadataPath): bool + { + $tilePattern = '/^' . preg_quote(basename($metadataPath, '-.metadata'), '/') . '-.*-\d+$/D'; + foreach (new DirectoryIterator(dirname($metadataPath)) as $file) { + if (preg_match($tilePattern, $file->getFilename()) === 1 && !$file->isLink() && $file->isFile()) { + return true; + } + } + + return false; + } + + private static function isExpired(string $path, int $cutoff): bool + { + clearstatcache(true, $path); + if (is_link($path)) { + return false; + } + $mtime = @filemtime($path); + return $mtime !== false && $mtime < $cutoff; + } + + private static function delete(string $path): void + { + if (!@unlink($path)) { + throw new RuntimeException('Could not delete cache file "' . $path . '"'); + } + } +} diff --git a/classes/FileLock.php b/classes/FileLock.php new file mode 100644 index 0000000..a43ba61 --- /dev/null +++ b/classes/FileLock.php @@ -0,0 +1,89 @@ += $deadline) { + throw $error; + } + } + } + + $handle = @fopen($path, $create ? 'c+b' : 'r+b'); + if ($handle === false) { + if (!$create) { + return null; + } + + // The parent may have been removed before fopen() and recreated before this check. + clearstatcache(true, dirname($path)); + if (hrtime(true) / 1e9 >= $deadline) { + throw new RuntimeException('Could not open lock file "' . $path . '"'); + } + } else { + if (flock($handle, LOCK_EX | LOCK_NB) && self::isCurrentFile($handle, $path)) { + return new self($handle); + } + + fclose($handle); + } + + $remaining = $deadline - hrtime(true) / 1e9; + if ($remaining <= 0) { + return null; + } + + usleep((int)min(50000, ceil($remaining * 1e6))); + } while (true); + } + + /** @param resource $handle */ + private static function isCurrentFile($handle, string $path): bool + { + clearstatcache(true, $path); + $current = @stat($path); + $opened = fstat($handle); + + // A cleaner may unlink metadata while another request has its old inode open. + return $current !== false && $opened !== false + && $current['dev'] === $opened['dev'] && $current['ino'] === $opened['ino']; + } + + public function release(): void + { + if (is_resource($this->handle)) { + fclose($this->handle); + $this->handle = null; + } + } + + public function __destruct() + { + $this->release(); + } +} diff --git a/tests/CacheLockTest.php b/tests/CacheLockTest.php new file mode 100644 index 0000000..a4d721f --- /dev/null +++ b/tests/CacheLockTest.php @@ -0,0 +1,260 @@ +worker('hold', $this->metadataPath(), 0, 3100000); + $this->assertSame("LOCKED\n", fgets($worker['pipes'][1])); + + $response = Base::handleTileRequest($this->config(4)); + + $this->assertSame('tile', $response->body); + $this->assertSame('tile', file_get_contents($this->tempDir . '/cache/0/0/0-source-0')); + } + + public function testConfiguredTimeoutIsBoundedAndClosesItsHandle(): void + { + $lock = FileLock::acquire($this->metadataPath(), 0); + $this->assertNotNull($lock); + $start = hrtime(true); + try { + Base::handleTileRequest($this->config(0.03)); + $this->fail('A competing request must time out'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('Can not lock', $error->getMessage()); + $this->assertLessThan(1, (hrtime(true) - $start) / 1e9); + } finally { + $lock->release(); + } + + $nextLock = FileLock::acquire($this->metadataPath(), 0); + $this->assertNotNull($nextLock); + $nextLock->release(); + } + + public function testPipelineFailureReleasesTheLock(): void + { + $cfg = $this->config(0); + $cfg['ops'][] = ['op' => 'unknown']; + try { + Base::handleTileRequest($cfg); + $this->fail('An unknown operation must fail'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('No op handler', $error->getMessage()); + } + + $lock = FileLock::acquire($this->metadataPath(), 0); + $this->assertNotNull($lock); + $lock->release(); + } + + public function testWaiterCannotUseAnUnlinkedMetadataInode(): void + { + $worker = $this->worker('wait', $this->metadataPath(), 0.3); + $this->assertSame("READY\n", fgets($worker['pipes'][1])); + $oldLock = FileLock::acquire($this->metadataPath(), 0); + $replacement = FileLock::acquire($this->tempDir . '/replacement', 0); + $this->assertNotNull($oldLock); + $this->assertNotNull($replacement); + fwrite($worker['pipes'][0], "GO\n"); + usleep(100000); + + // Atomically replace the locked inode, as cleanup followed by a new request can do. + rename($this->tempDir . '/replacement', $this->metadataPath()); + $oldLock->release(); + $this->assertSame("TIMEOUT\n", fgets($worker['pipes'][1])); + // Wait for process exit before checking that a new request can acquire the lock. + $this->assertSame('', stream_get_contents($worker['pipes'][1])); + $replacement->release(); + + $this->assertNotNull(FileLock::acquire($this->metadataPath(), 0)); + } + + public function testZeroTimeoutDoesNotWaitAndMissingReadOnlyLockIsNotCreated(): void + { + $this->assertNull(FileLock::acquire($this->tempDir . '/missing', 0, false)); + $this->assertFileDoesNotExist($this->tempDir . '/missing'); + $lock = FileLock::acquire($this->metadataPath(), 0); + $this->assertNotNull($lock); + $this->assertNull(FileLock::acquire($this->metadataPath(), 0)); + $lock->release(); + } + + #[DataProvider('directoryRecreation')] + public function testRetriesWhenDirectoryIsPrunedDuringCreation(bool $recreateAfterFailedOpen): void + { + $path = $this->metadataPath(); + $directory = dirname($path); + mkdir($directory, 0777, true); + $removed = false; + $recreated = false; + set_error_handler(static function (int $severity, string $message) use ( + $directory, $recreateAfterFailedOpen, &$removed, &$recreated + ): bool { + if (!$removed && str_contains($message, 'mkdir(): File exists')) { + // Remove the empty directory between mkdir() failing and mkdirp() checking is_dir(). + rmdir($directory); + clearstatcache(true, $directory); + $removed = true; + return true; + } + + if ($recreateAfterFailedOpen && $removed && !$recreated && str_contains($message, 'fopen(')) { + // Another request recreates the parent after the open fails, before its error is checked. + mkdir($directory, 0777, true); + clearstatcache(true, $directory); + $recreated = true; + return true; + } + + return false; + }); + + try { + $lock = FileLock::acquire($path, 0.2); + } finally { + restore_error_handler(); + } + + $this->assertTrue($removed); + $this->assertSame($recreateAfterFailedOpen, $recreated); + $this->assertNotNull($lock); + $lock->release(); + $this->assertFileExists($path); + } + + public static function directoryRecreation(): array + { + return [ + 'recreated on retry' => [false], + 'recreated after failed open' => [true], + ]; + } + + public function testDirectoryCreationFailureIsReportedWithinTheLockBudget(): void + { + $parentFile = $this->tempDir . '/not-a-directory'; + file_put_contents($parentFile, 'blocked'); + $timeout = 0.03; + $start = hrtime(true); + + try { + FileLock::acquire($parentFile . '/metadata', $timeout); + $this->fail('A file cannot be used as the metadata directory'); + } catch (RuntimeException $error) { + $this->assertStringContainsString($parentFile, $error->getMessage()); + $elapsed = (hrtime(true) - $start) / 1e9; + $this->assertGreaterThanOrEqual($timeout, $elapsed); + $this->assertLessThan(1, $elapsed); + } + } + + #[DataProvider('invalidTimeouts')] + public function testInvalidTimeoutIsRejected(float $timeout): void + { + $this->expectException(InvalidArgumentException::class); + FileLock::acquire($this->metadataPath(), $timeout); + } + + public static function invalidTimeouts(): array + { + return [[-1.0], [INF], [NAN]]; + } + + public function testCacheHitUpdatesMetadataAccessTimeWithoutRefreshingTile(): void + { + Base::handleTileRequest($this->config(0)); + $tilePath = $this->tempDir . '/cache/0/0/0-source-0'; + $oldTime = time() - 60; + touch($tilePath, $oldTime); + touch($this->metadataPath(), $oldTime); + unlink($this->tempDir . '/tile'); + clearstatcache(); + + $this->assertSame('tile', Base::handleTileRequest($this->config(0))->body); + clearstatcache(); + $this->assertSame($oldTime, filemtime($tilePath)); + $this->assertGreaterThan($oldTime, filemtime($this->metadataPath())); + } + + private function config(float $timeout): array + { + return [ + 'cacheServerPath' => $this->tempDir . '/cache', + 'cacheLockTimeout' => $timeout, + 'ops' => [[ + 'cacheServerName' => 'source', + 'urls' => ['file://' . $this->tempDir . '/tile'], + 'mimeType' => 'image/png', + 'cacheBrowserTtl' => 60, + 'cacheServerTtl' => 3600, + ]], + ]; + } + + private function metadataPath(): string + { + return $this->tempDir . '/cache/0/0/0-.metadata'; + } + + private function worker(string $mode, string $path, float $timeout, int $holdMicroseconds = 0): array + { + $process = proc_open( + [PHP_BINARY, __DIR__ . '/fixtures/cache-lock-worker.php', $mode, $path, (string)$timeout, (string)$holdMicroseconds], + [['pipe', 'r'], ['pipe', 'w'], ['pipe', 'w']], + $pipes + ); + $this->assertIsResource($process); + stream_set_timeout($pipes[1], 10); + $worker = ['process' => $process, 'pipes' => $pipes]; + $this->workers[] = $worker; + return $worker; + } + + protected function setUp(): void + { + $this->tempDir = sys_get_temp_dir() . '/tile_proxy_lock_' . bin2hex(random_bytes(8)); + mkdir($this->tempDir); + file_put_contents($this->tempDir . '/tile', 'tile'); + $this->savedGet = $_GET; + $_GET = ['z' => '0', 'x' => '0', 'y' => '0']; + } + + protected function tearDown(): void + { + foreach ($this->workers as $worker) { + fclose($worker['pipes'][0]); + $errors = stream_get_contents($worker['pipes'][2]); + fclose($worker['pipes'][1]); + fclose($worker['pipes'][2]); + $status = proc_close($worker['process']); + $this->assertSame(0, $status, $errors); + } + $_GET = $this->savedGet; + $files = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($this->tempDir, RecursiveDirectoryIterator::SKIP_DOTS), + RecursiveIteratorIterator::CHILD_FIRST + ); + foreach ($files as $file) { + $file->isDir() ? rmdir($file->getPathname()) : unlink($file->getPathname()); + } + rmdir($this->tempDir); + } +} diff --git a/tests/CachePrunerTest.php b/tests/CachePrunerTest.php new file mode 100644 index 0000000..bcf589b --- /dev/null +++ b/tests/CachePrunerTest.php @@ -0,0 +1,389 @@ +write('dark/12/2048/1024-' . $namespace . '-0', 120); + $this->write('dark/12/2048/1024-overlay-0', 120); + $this->write('dark/12/2048/1024-' . $namespace . '-1', 120); + $this->write('dark/12/2048/1024-.metadata', 120); + + $this->assertSame(4, CachePruner::prune($this->tempDir, 60)); + $this->assertDirectoryDoesNotExist($this->tempDir . '/dark'); + $this->assertDirectoryExists($this->tempDir); + $this->assertSame(0, CachePruner::prune($this->tempDir, 60)); + } + + #[DataProvider('cacheNamespaces')] + public function testRemovesObsoleteNamespacesButKeepsFreshTileAndItsMetadata(string $namespace): void + { + $old = $this->write('1/0/0-old-url-hash-0', 120); + $fresh = $this->write('1/0/0-' . $namespace . '-0'); + $metadata = $this->write('1/0/0-.metadata', 120); + + $this->assertSame(1, CachePruner::prune($this->tempDir, 60)); + $this->assertFileDoesNotExist($old); + $this->assertFileExists($fresh); + $this->assertFileExists($metadata); + } + + public static function cacheNamespaces(): array + { + return [ + 'named namespace' => ['current-url-hash'], + 'empty namespace' => [''], + ]; + } + + public function testSkipsTilesLockedByARequestThenPrunesAfterRelease(): void + { + $tile = $this->write('1/0/0-source-0', 120); + $metadata = $this->write('1/0/0-.metadata', 120); + $lock = FileLock::acquire($metadata, 0); + $this->assertNotNull($lock); + + $this->assertSame(0, CachePruner::prune($this->tempDir, 60)); + $this->assertFileExists($tile); + $lock->release(); + $this->assertSame(2, CachePruner::prune($this->tempDir, 60)); + } + + public function testKeepsRecentlyRequestedMetadataEvenWhenAllTilesAreOld(): void + { + $this->write('1/0/0-source-0', 120); + $metadata = $this->write('1/0/0-.metadata'); + $this->write('1/0/1-.metadata', 120); + + $this->assertSame(2, CachePruner::prune($this->tempDir, 60)); + $this->assertFileExists($metadata); + } + + #[DataProvider('staleScans')] + public function testKeepsMetadataForCheckpointWrittenAfterScanByAFailedRequest( + bool $scanHadTile, string $checkpointNamespace + ): void + { + $metadata = $this->write('cache/1/0/0-.metadata', 120); + $scannedTiles = $scanHadTile ? [$this->write('cache/1/0/0-old-source-0', 120)] : []; + $source = $this->write('source'); + $metadataMtime = filemtime($metadata); + $savedGet = $_GET; + $_GET = ['z' => '1', 'x' => '0', 'y' => '0']; + + try { + Base::handleTileRequest([ + 'cacheServerPath' => $this->tempDir . '/cache', + 'cacheLockTimeout' => 0, + 'ops' => [ + [ + 'cacheServerName' => $checkpointNamespace, + 'urls' => ['file://' . $source], + 'mimeType' => 'image/png', + 'cacheBrowserTtl' => 60, + 'cacheServerTtl' => 3600, + ], + // Merge checkpoints the source before this invalid subpipeline fails. + ['op' => 'merge', 'ops' => [[]]], + ], + ]); + $this->fail('The merge subpipeline must fail after checkpointing the source'); + } catch (RuntimeException $error) { + $this->assertStringContainsString('Missing `cacheServerName`', $error->getMessage()); + } finally { + $_GET = $savedGet; + } + + $checkpoint = $this->tempDir . '/cache/1/0/0-' . $checkpointNamespace . '-0'; + $this->assertFileExists($checkpoint); + clearstatcache(true, $metadata); + $this->assertSame($metadataMtime, filemtime($metadata)); + + // Resume pruning with the pre-request scan to reproduce the interleaving without timing races. + $pruneTile = new ReflectionMethod(CachePruner::class, 'pruneTile'); + $this->assertSame($scanHadTile ? 1 : 0, $pruneTile->invoke(null, $metadata, $scannedTiles, time() - 60)); + $this->assertFileExists($checkpoint); + $this->assertFileExists($metadata); + + // Keeping metadata lets a later retention run remove the checkpoint once it ages out. + touch($checkpoint, time() - 120); + $this->assertSame(2, CachePruner::prune($this->tempDir, 60)); + $this->assertFileDoesNotExist($checkpoint); + $this->assertFileDoesNotExist($metadata); + } + + public static function staleScans(): array + { + return [ + 'metadata only' => [false, 'new-source'], + 'obsolete namespace' => [true, 'new-source'], + 'metadata only with empty checkpoint namespace' => [false, ''], + 'obsolete namespace with empty checkpoint namespace' => [true, ''], + ]; + } + + #[DataProvider('cacheRootSuffixes')] + public function testSharedCachePruningPreservesAMapboxAssetWriteInProgress(string $rootSuffix): void + { + $this->write('1/0/0-source-0', 120); + $this->write('1/0/0-.metadata', 120); + $style = '{"version":8,"sources":{},"layers":[]}'; + $upstreamStyle = $this->write('upstream-style.json'); + file_put_contents($upstreamStyle, $style); + $mapboxDirectory = $this->tempDir . '/mapbox-style-proxy/example/style'; + mkdir($mapboxDirectory, 0777, true); + $cacheRoot = $this->tempDir . $rootSuffix; + $pruned = false; + $deleted = null; + + set_error_handler(static function (int $severity, string $message) use ($cacheRoot, &$pruned, &$deleted): bool { + if (!$pruned && str_contains($message, 'mkdir(): File exists')) { + // Pause the asset writer in mkdirp(), while its existing directory is still empty. + $pruned = true; + $deleted = CachePruner::prune($cacheRoot, 60); + return true; + } + + return false; + }); + + try { + $response = MapboxStyleProxy::handleRequest([ + 'cacheServerPath' => $cacheRoot, + 'styles' => ['example' => [ + 'upstreamStyleUrl' => 'file://' . $upstreamStyle, + 'allowedSchemes' => ['file'], + 'allowedPathPrefixes' => [$this->tempDir . '/'], + ]], + ], '/styles/example.json'); + } finally { + restore_error_handler(); + } + + $this->assertTrue($pruned); + $this->assertSame(2, $deleted); + $this->assertSame(8, json_decode($response->body, true)['version']); + $this->assertDirectoryExists($mapboxDirectory); + $this->assertSame($style, file_get_contents($mapboxDirectory . '/' . sha1('file://' . $upstreamStyle))); + } + + #[DataProvider('cacheRootSuffixes')] + public function testIgnoresFilesAndEmptyDirectoriesInReservedMapboxCacheTree(string $rootSuffix): void + { + $asset = $this->write('mapbox-style-proxy/example/style/abc', 120); + $tile = $this->write('mapbox-style-proxy/example/tile/1-source-0', 120); + $metadata = $this->write('mapbox-style-proxy/example/tile/1-.metadata', 120); + $emptyDirectory = $this->tempDir . '/mapbox-style-proxy/example/glyph'; + mkdir($emptyDirectory); + + $this->assertSame(0, CachePruner::prune($this->tempDir . $rootSuffix, 60)); + foreach ([$asset, $tile, $metadata] as $path) { + $this->assertFileExists($path); + } + $this->assertDirectoryExists($emptyDirectory); + } + + public static function cacheRootSuffixes(): array + { + return [ + 'no trailing slash' => [''], + 'one trailing slash' => ['/'], + 'two trailing slashes' => ['//'], + 'three trailing slashes' => ['///'], + ]; + } + + public function testIgnoresSymlinksUnrelatedFilesAndOtherTileCoordinates(): void + { + $this->write('1/0/1-.metadata', 120); + $this->write('1/0/1-source-0', 120); + $other = $this->write('1/0/10-source-0'); + $this->write('1/0/10-.metadata'); + $unrelated = $this->write('1/0/notes.txt', 120); + $asset = $this->write('map-assets/abc.json', 120); + $target = $this->write('target', 120); + $this->write('1/0/2-.metadata', 120); + symlink($target, $this->tempDir . '/1/0/2-source-0'); + symlink($this->tempDir . '/1', $this->tempDir . '/linked'); + + $this->assertSame(3, CachePruner::prune($this->tempDir, 60)); + foreach ([$other, $unrelated, $asset, $target, $this->tempDir . '/1/0/2-source-0'] as $path) { + $this->assertFileExists($path); + } + $this->assertTrue(is_link($this->tempDir . '/linked')); + } + + public function testMissingCacheIsANoOp(): void + { + $this->assertSame(0, CachePruner::prune($this->tempDir . '/missing', 60)); + $this->assertDirectoryDoesNotExist($this->tempDir . '/missing'); + } + + #[DataProvider('pruneEntryPoints')] + public function testDirectoryRemovedByAnotherProcessIsANoOpDespiteCachedStat(bool $pruneRoot): void + { + if (!defined('SIGSTOP') || !defined('SIGCONT')) { + $this->markTestSkipped('Process signals are required to preserve the parent stat cache'); + } + + $directory = $this->tempDir . '/removed'; + mkdir($directory); + $pruneDirectory = new ReflectionMethod(CachePruner::class, 'pruneDirectory'); + + // A subprocess removes the directory without clearing this process's positive stat cache. + $process = proc_open( + [PHP_BINARY, '-r', 'fgets(STDIN); exit(rmdir($argv[1]) ? 0 : 1);', $directory], + [['pipe', 'r'], ['pipe', 'w'], ['pipe', 'w']], + $pipes + ); + $this->assertIsResource($process); + + try { + proc_terminate($process, SIGSTOP); + $deadline = hrtime(true) + 2e9; + do { + $status = proc_get_status($process); + if ($status['stopped'] || !$status['running'] || hrtime(true) >= $deadline) { + break; + } + usleep(1000); + } while (true); + $this->assertTrue($status['stopped']); + fwrite($pipes[0], "GO\n"); + $this->assertTrue(is_dir($directory)); + + // Signals and process-status polling avoid pipe I/O, which clears PHP's stat cache. + proc_terminate($process, SIGCONT); + $deadline = hrtime(true) + 2e9; + do { + $status = proc_get_status($process); + if (!$status['running'] || hrtime(true) >= $deadline) { + break; + } + usleep(1000); + } while (true); + $this->assertTrue(is_dir($directory), 'The parent must retain the stale positive directory stat'); + + $deleted = $pruneRoot + ? CachePruner::prune($directory, 60) + : $pruneDirectory->invoke(null, $directory, time() - 60); + $this->assertSame(0, $deleted); + $this->assertFalse($status['running']); + $this->assertSame(0, $status['exitcode']); + $this->assertDirectoryDoesNotExist($directory); + } finally { + proc_terminate($process, SIGCONT); + fclose($pipes[0]); + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($process); + } + } + + public static function pruneEntryPoints(): array + { + return [ + 'directory sweep' => [false], + 'cache root' => [true], + ]; + } + + #[DataProvider('directoryIteratorClasses')] + public function testRetriesOpeningADirectoryRecreatedAfterTheFailedOpen(string $iteratorClass): void + { + $directory = $this->tempDir . '/recreated'; + $recreated = false; + $open = static function () use ($directory, $iteratorClass, &$recreated): DirectoryIterator { + try { + return new $iteratorClass($directory); + } catch (UnexpectedValueException $error) { + // A request restores the directory between its failed open and the pruner's check. + mkdir($directory); + $recreated = true; + throw $error; + } + }; + + $openDirectory = new ReflectionMethod(CachePruner::class, 'openDirectory'); + $iterator = $openDirectory->invoke(null, $directory, $open); + + $this->assertTrue($recreated); + $this->assertInstanceOf($iteratorClass, $iterator); + $this->assertDirectoryExists($directory); + } + + public static function directoryIteratorClasses(): array + { + return [ + 'directory sweep' => [DirectoryIterator::class], + 'cache root' => [RecursiveDirectoryIterator::class], + ]; + } + + public function testPersistentDirectoryOpenFailuresAreReported(): void + { + $open = static function (): DirectoryIterator { + throw new UnexpectedValueException('Persistent directory open failure'); + }; + $openDirectory = new ReflectionMethod(CachePruner::class, 'openDirectory'); + + $this->expectException(UnexpectedValueException::class); + $this->expectExceptionMessage('Persistent directory open failure'); + $openDirectory->invoke(null, $this->tempDir, $open); + } + + public function testNonPositiveRetentionIsRejected(): void + { + $this->expectException(InvalidArgumentException::class); + CachePruner::prune($this->tempDir, 0); + } + + private function write(string $relativePath, int $age = 0): string + { + $path = $this->tempDir . '/' . $relativePath; + if (!is_dir(dirname($path))) { + mkdir(dirname($path), 0777, true); + } + file_put_contents($path, '{}'); + touch($path, time() - $age); + return $path; + } + + protected function setUp(): void + { + $this->tempDir = sys_get_temp_dir() . '/tile_proxy_pruner_' . bin2hex(random_bytes(8)); + mkdir($this->tempDir); + } + + protected function tearDown(): void + { + $files = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($this->tempDir, RecursiveDirectoryIterator::SKIP_DOTS), + RecursiveIteratorIterator::CHILD_FIRST + ); + foreach ($files as $file) { + $file->isDir() && !$file->isLink() ? rmdir($file->getPathname()) : unlink($file->getPathname()); + } + rmdir($this->tempDir); + } +} diff --git a/tests/fixtures/cache-lock-worker.php b/tests/fixtures/cache-lock-worker.php new file mode 100644 index 0000000..d2d1883 --- /dev/null +++ b/tests/fixtures/cache-lock-worker.php @@ -0,0 +1,25 @@ +release(); +}