8cd5e10038
* feat: alternate search index for nimbus
Create an alternate search index for Nimbus that filters out
feature-flagged pages (equivalent to setting all feature flags to
false).
Notes:
- Creates two new DB tables, `page_nimbus` and `page_section_nimbus`,
which are filtered versions of `page` and `page_section`
- Makes `nimbus` versions of all the DB search functions
- Refactored the embedding upload script. Changes to make it faster (got
annoyed by how slow it was when testing...), incorporate retries, and
produce better summary logs.
- Upload script, when run with the environment variable
ENABLED_FEATURES_OVERRIDE_DISABLE_ALL, produces and uploads the
alternate search index
- Changed all the search calls in frontend/API to check for
`isFeatureEnabled('search:fullIndex')` to determine whether to search
the full or alternate index
* ci: produce nimbus search indexes on merge
* fix: turn full search index on
80 lines
2.0 KiB
YAML
80 lines
2.0 KiB
YAML
name: Generate Embeddings for Search
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
paths:
|
|
- '.github/workflows/search.yml'
|
|
- 'supabase/migrations/**'
|
|
- 'apps/docs/**'
|
|
workflow_dispatch:
|
|
inputs:
|
|
refresh:
|
|
description: 'Refresh all pages'
|
|
required: false
|
|
type: boolean
|
|
schedule:
|
|
- cron: '0 0 * * *'
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-prod
|
|
cancel-in-progress: false
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
deploy:
|
|
runs-on: blacksmith-4vcpu-ubuntu-2404
|
|
|
|
env:
|
|
DOCS_GITHUB_APP_ID: ${{ secrets.SEARCH_GITHUB_APP_ID }}
|
|
DOCS_GITHUB_APP_INSTALLATION_ID: ${{ secrets.SEARCH_GITHUB_APP_INSTALLATION_ID }}
|
|
DOCS_GITHUB_APP_PRIVATE_KEY: ${{ secrets.SEARCH_GITHUB_APP_PRIVATE_KEY }}
|
|
NEXT_PUBLIC_MISC_URL: ${{ secrets.NEXT_PUBLIC_MISC_URL}}
|
|
NEXT_PUBLIC_MISC_ANON_KEY: ${{ secrets.NEXT_PUBLIC_MISC_ANON_KEY }}
|
|
NEXT_PUBLIC_SUPABASE_URL: ${{ secrets.SEARCH_SUPABASE_URL }}
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
SUPABASE_SECRET_KEY: ${{ secrets.SEARCH_SUPABASE_SERVICE_ROLE_KEY }}
|
|
|
|
steps:
|
|
- name: Check out repo
|
|
uses: actions/checkout@v4
|
|
with:
|
|
sparse-checkout: |
|
|
apps/docs
|
|
apps/www/.env.local.example
|
|
examples
|
|
packages
|
|
supabase
|
|
|
|
- uses: pnpm/action-setup@v4
|
|
name: Install pnpm
|
|
with:
|
|
run_install: false
|
|
|
|
- name: Setup node
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version-file: '.nvmrc'
|
|
|
|
- name: Download dependencies
|
|
run: pnpm i
|
|
|
|
- name: Update embeddings
|
|
working-directory: ./apps/docs
|
|
if: ${{ !inputs.refresh }}
|
|
run: |
|
|
pnpm run codegen:examples
|
|
pnpm run embeddings
|
|
pnpm run embeddings:nimbus
|
|
|
|
- name: Refresh embeddings
|
|
working-directory: ./apps/docs
|
|
if: ${{ inputs.refresh }}
|
|
run: |
|
|
pnpm run codegen:examples
|
|
pnpm run embeddings:refresh
|
|
pnpm run embeddings:nimbus:refresh
|