Improve Upstash Search indexing

Author: webstackdev

.github/workflows/search.yml

@@ -46,12 +46,6 @@ jobs:
           current_run_id: ${{ github.event.workflow_run.id }}
           head_sha: ${{ github.event.workflow_run.head_sha }}
 
-      - name: Setup Node.js
-        if: steps.scope.outputs.should_index == 'true'
-        uses: actions/setup-node@v6.3.0
-        with:
-          node-version: '24.x'
-
       - name: Validate Upstash credentials
         if: steps.scope.outputs.should_index == 'true'
         run: |
@@ -64,14 +58,23 @@ jobs:
             exit 1
           fi
 
-      - name: Crawl /articles
-        if: steps.scope.outputs.should_index == 'true' && steps.scope.outputs.crawl_articles == 'true'
-        run: >-
-          npx -y @upstash/search-crawler
-          --upstash-url "${{ vars.UPSTASH_SEARCH_REST_URL }}"
-          --upstash-token "${{ secrets.UPSTASH_SEARCH_REST_TOKEN }}"
-          --index-name "default"
-          --doc-url "https://www.webstackbuilders.com/articles"
+      - name: Index changed collections
+        if: steps.scope.outputs.should_index == 'true'
+        env:
+          UPSTASH_SEARCH_REST_URL: ${{ vars.UPSTASH_SEARCH_REST_URL }}
+          UPSTASH_SEARCH_REST_TOKEN: ${{ secrets.UPSTASH_SEARCH_REST_TOKEN }}
+        run: |
+          COLLECTIONS=""
+          if [ "${{ steps.scope.outputs.crawl_articles }}" = "true" ]; then
+            COLLECTIONS="$COLLECTIONS --collection articles --collection deep-dive"
+          fi
+          if [ "${{ steps.scope.outputs.crawl_services }}" = "true" ]; then
+            COLLECTIONS="$COLLECTIONS --collection services"
+          fi
+          if [ "${{ steps.scope.outputs.crawl_case_studies }}" = "true" ]; then
+            COLLECTIONS="$COLLECTIONS --collection case-studies"
+          fi
+          python3 scripts/search-index.py --no-drop $COLLECTIONS
 
       - name: Prune removed /articles
         if: steps.scope.outputs.should_index == 'true' && steps.scope.outputs.crawl_articles == 'true'
@@ -88,15 +91,6 @@ jobs:
           site_origin: https://www.webstackbuilders.com
           collection: articles
 
-      - name: Crawl /services
-        if: steps.scope.outputs.should_index == 'true' && steps.scope.outputs.crawl_services == 'true'
-        run: >-
-          npx -y @upstash/search-crawler
-          --upstash-url "${{ vars.UPSTASH_SEARCH_REST_URL }}"
-          --upstash-token "${{ secrets.UPSTASH_SEARCH_REST_TOKEN }}"
-          --index-name "default"
-          --doc-url "https://www.webstackbuilders.com/services"
-
       - name: Prune removed /services
         if: steps.scope.outputs.should_index == 'true' && steps.scope.outputs.crawl_services == 'true'
         uses: ./.github/actions/prune-upstash-search
@@ -112,15 +106,6 @@ jobs:
           site_origin: https://www.webstackbuilders.com
           collection: services
 
-      - name: Crawl /case-studies
-        if: steps.scope.outputs.should_index == 'true' && steps.scope.outputs.crawl_case_studies == 'true'
-        run: >-
-          npx -y @upstash/search-crawler
-          --upstash-url "${{ vars.UPSTASH_SEARCH_REST_URL }}"
-          --upstash-token "${{ secrets.UPSTASH_SEARCH_REST_TOKEN }}"
-          --index-name "default"
-          --doc-url "https://www.webstackbuilders.com/case-studies"
-
       - name: Prune removed /case-studies
         if: steps.scope.outputs.should_index == 'true' && steps.scope.outputs.crawl_case_studies == 'true'
         uses: ./.github/actions/prune-upstash-search

_TODO.md

@@ -33,10 +33,6 @@ Discuss agentic AI integrations to add to Backstage
 
 Implement mitigations in test/e2e/specs/07-performance/PERFORMANCE.md
 
-## Email Templates
-
-Right now we're using string literals to define HTML email templates for site mails. We should use Nunjucks with the rule-checking for valid CSS in HTML emails like we have in the corporate email footer repo.
-
 ## Chat bot tying into my phone and email
 
 Vercel AI Gateway, maybe could use for a chatbot:
@@ -160,18 +156,22 @@ Styling: You must use inline CSS within your `headerTemplate` or `footerTemplate
 
 ## ToolTips
 
-Need a tooltip component for consistency. List to add tooltips to:
+Need a tooltip component for consistency.
+
+List to add tooltips to:
 
 - Themepicker button
 - Search button
+
+These have tooltips, how are they being generated?
+
 - Abbreviations in markdown
 - "Report a Bug" in footer
 - RSS feed icon in footer
 
 ## Support Pages to Style
 
-- Bug reporter modal
-- Email templates
+- Bug reporter modal. Should  have option to minimize and carry state if it's minimized after someone enters bug info in case they need to collect info from the site for the report.
 
 ## HubSpot Signup Issues
 

requirements.txt

@@ -7,5 +7,6 @@ pytest-sugar==1.1.1
 pytest-xdist==3.8.0
 pytest==9.0.2
 python-dotenv==1.2.2
+PyYAML==6.0.2
 requests==2.32.5
 upstash-search==0.1.1