Generate LLMs text during site build

.github/workflows/jekyll.yml

@@ -0,0 +1,60 @@
+name: Deploy Jekyll site to Pages
+
+on:
+  push:
+    branches: ["master"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.2.2'
+          bundler-cache: true
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.3'
+      - name: Setup Pages
+        id: pages
+        uses: actions/configure-pages@v5
+      - name: Run custom scripts
+        run: |
+          ruby bin/merge_md_files.rb
+          php bin/generate_llms_full.php
+      - name: Build with Jekyll
+        run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}"
+        env:
+          JEKYLL_ENV: production
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+
+  deploy:
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-22.04
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -1,2 +1,3 @@
 _site/
 watch
+llms-full.txt

bin/generate_llms_full.php

@@ -0,0 +1,83 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * Generate llms-full.txt from llms.txt and English manual markdown files.
+ */
+
+$baseDir = dirname(__DIR__);
+$llmsFile = $baseDir . '/llms.txt';
+$manualDir = $baseDir . '/manuals/1.0/en';
+$outputFile = $baseDir . '/llms-full.txt';
+
+if (! is_file($llmsFile)) {
+    fwrite(STDERR, "llms.txt not found at {$llmsFile}\n");
+    exit(1);
+}
+
+if (! is_dir($manualDir)) {
+    fwrite(STDERR, "Manual directory not found at {$manualDir}\n");
+    exit(1);
+}
+
+$skipFiles = [
+    '1page.md' => true,
+    'onepage.md' => true,
+    'ai-assistant.md' => true,
+];
+
+$files = [];
+$iterator = new RecursiveIteratorIterator(
+    new RecursiveDirectoryIterator($manualDir, FilesystemIterator::SKIP_DOTS),
+);
+
+foreach ($iterator as $file) {
+    if (! $file instanceof SplFileInfo || $file->getExtension() !== 'md') {
+        continue;
+    }
+
+    if (isset($skipFiles[$file->getBasename()])) {
+        continue;
+    }
+
+    $files[] = $file->getPathname();
+}
+
+sort($files, SORT_STRING);
+
+$llmsContent = file_get_contents($llmsFile);
+if ($llmsContent === false) {
+    fwrite(STDERR, "Failed to read llms.txt at {$llmsFile}\n");
+    exit(1);
+}
+
+$content = rtrim($llmsContent) . "\n\n---\n\n# Full Documentation\n\n";
+
+foreach ($files as $file) {
+    $markdown = file_get_contents($file);
+    if ($markdown === false) {
+        fwrite(STDERR, "Failed to read markdown file at {$file}\n");
+        exit(1);
+    }
+
+    $markdown = preg_replace('/\A---\s*\R.*?\R---\s*\R/ms', '', $markdown, 1) ?? $markdown;
+    $markdown = trim($markdown);
+
+    if ($markdown === '') {
+        continue;
+    }
+
+    $relativePath = ltrim(str_replace($baseDir, '', $file), '/');
+    $content .= "<!-- Source: {$relativePath} -->\n\n";
+    $content .= $markdown . "\n\n";
+}
+
+$bytesWritten = file_put_contents($outputFile, $content);
+if ($bytesWritten === false || $bytesWritten !== strlen($content)) {
+    fwrite(STDERR, "Failed to write complete llms-full.txt to {$outputFile}\n");
+    exit(1);
+}
+
+echo "Generated llms-full.txt successfully.\n";
+echo 'Included files: ' . count($files) . "\n";

bin/serve_local.sh

@@ -1,6 +1,11 @@
 #!/bin/bash
+set -euo pipefail
+trap 'echo "serve_local.sh failed; check merge_md_files.rb, generate_llms_full.php, or Jekyll output above." >&2' ERR
+
 # This script is used to serve the Jekyll site locally with automatic rebuilding.
 # 'bundle exec' ensures we're using the correct versions of each gem according to our Gemfile.lock.
 # 'jekyll serve' starts a Jekyll development server.
 # '--watch' option automatically rebuilds the site when files are modified.
+ruby bin/merge_md_files.rb
+php bin/generate_llms_full.php
 bundle exec jekyll serve --watch --trace