From dfda9da26d08d77425130423238ca523ec162046 Mon Sep 17 00:00:00 2001 From: Charis <26616127+charislam@users.noreply.github.com> Date: Thu, 19 Sep 2024 14:28:07 -0400 Subject: [PATCH] feat(docs): add revalidation api route (#29378) Add a route for manually revalidating cache contents by tag. The route is protected by authentication to prevent abuse. Automated actions in CI should be set up with a basic API key, which has a rate limit of 6 hours between changes. Overriding is possible with an override key, which should be used as an escape hatch. Usage: - API key provided in header `Authorization: Bearer ` - Body has shape `{ tags: string[] }` --- apps/docs/app/api/revalidate/route.test.ts | 188 +++++++++++ apps/docs/app/api/revalidate/route.ts | 116 +++++++ packages/common/database-types.ts | 297 ++++++++++++++++-- packages/common/package.json | 1 + .../20240918220938_validation_history.sql | 24 ++ turbo.json | 10 +- 6 files changed, 612 insertions(+), 24 deletions(-) create mode 100644 apps/docs/app/api/revalidate/route.test.ts create mode 100644 apps/docs/app/api/revalidate/route.ts create mode 100644 supabase/migrations/20240918220938_validation_history.sql diff --git a/apps/docs/app/api/revalidate/route.test.ts b/apps/docs/app/api/revalidate/route.test.ts new file mode 100644 index 0000000000..befeabd43e --- /dev/null +++ b/apps/docs/app/api/revalidate/route.test.ts @@ -0,0 +1,188 @@ +/* eslint-disable turbo/no-undeclared-env-vars */ + +import { createClient } from '@supabase/supabase-js' +import { revalidateTag } from 'next/cache' +import { headers } from 'next/headers' +import { NextRequest } from 'next/server' +import { describe, it, expect, vi, beforeEach, afterEach, type Mock } from 'vitest' + +import { _handleRevalidateRequest } from './route' + +// Mock Next.js modules +vi.mock('next/cache', () => ({ + revalidateTag: vi.fn(), +})) + +vi.mock('next/headers', () => ({ + headers: vi.fn(), +})) + +// Mock Supabase client +vi.mock('@supabase/supabase-js', () => ({ + createClient: vi.fn(), +})) + +describe('_handleRevalidateRequest', () => { + let mockDate: Date + let originalEnv: NodeJS.ProcessEnv + let mockSupabaseClient: { + rpc: Mock + from: Mock + } + + beforeEach(() => { + // Store the original environment + originalEnv = { ...process.env } + + // Mock environment variables + process.env.DOCS_REVALIDATION_KEYS = 'basic_key' + process.env.DOCS_REVALIDATION_OVERRIDE_KEYS = 'override_key,other_override_key' + process.env.NEXT_PUBLIC_SUPABASE_URL = 'http://localhost:3000' + process.env.SUPABASE_SECRET_KEY = 'secret_key' + + // Mock current date + mockDate = new Date('2023-01-01T12:00:00Z') + vi.setSystemTime(mockDate) + + // Setup mock Supabase client + mockSupabaseClient = { + rpc: vi.fn(), + from: vi.fn(() => ({ + insert: vi.fn().mockResolvedValue({ error: null }), + })), + } + vi.mocked(createClient).mockReturnValue(mockSupabaseClient as any) + }) + + afterEach(() => { + process.env = originalEnv + vi.clearAllMocks() + vi.useRealTimers() + }) + + it('should return 401 if Authorization header is missing', async () => { + const request = new NextRequest('https://example.com', { + method: 'POST', + }) + + vi.mocked(headers).mockReturnValue(new Headers(request.headers)) + + const response = await _handleRevalidateRequest(request) + expect(response.status).toBe(401) + expect(await response.text()).toBe('Missing Authorization header') + }) + + it('should return 401 if Authorization header is invalid', async () => { + const request = new NextRequest('https://example.com', { + method: 'POST', + headers: { + Authorization: 'Bearer invalid_token', + }, + }) + + vi.mocked(headers).mockReturnValue(new Headers(request.headers)) + + const response = await _handleRevalidateRequest(request) + expect(response.status).toBe(401) + expect(await response.text()).toBe('Invalid Authorization header') + }) + + it('should return 400 if request body is malformed', async () => { + const request = new NextRequest('https://example.com', { + method: 'POST', + headers: { + Authorization: 'Bearer basic_key', + }, + body: JSON.stringify({ invalid: 'body' }), + }) + + vi.mocked(headers).mockReturnValue(new Headers(request.headers)) + + const response = await _handleRevalidateRequest(request) + expect(response.status).toBe(400) + expect(await response.text()).toContain('Malformed request body') + }) + + it('should revalidate tags if request is valid with basic permissions', async () => { + const request = new NextRequest('https://example.com', { + method: 'POST', + headers: { + Authorization: 'Bearer basic_key', + }, + body: JSON.stringify({ tags: ['tag1', 'tag2'] }), + }) + + vi.mocked(headers).mockReturnValue(new Headers(request.headers)) + + mockSupabaseClient.rpc.mockResolvedValue({ data: [] }) + + const response = await _handleRevalidateRequest(request) + expect(response.status).toBe(204) + expect(revalidateTag).toHaveBeenCalledTimes(2) + expect(revalidateTag).toHaveBeenCalledWith('tag1') + expect(revalidateTag).toHaveBeenCalledWith('tag2') + }) + + it('should return 429 if last revalidation was less than 6 hours ago with basic permissions', async () => { + const request = new NextRequest('https://example.com', { + method: 'POST', + headers: { + Authorization: 'Bearer basic_key', + }, + body: JSON.stringify({ tags: ['tag1'] }), + }) + + vi.mocked(headers).mockReturnValue(new Headers(request.headers)) + + const fiveHoursAgo = new Date(mockDate.getTime() - 5 * 60 * 60 * 1000) + mockSupabaseClient.rpc.mockResolvedValue({ + data: [{ created_at: fiveHoursAgo.toISOString() }], + }) + + const response = await _handleRevalidateRequest(request) + expect(response.status).toBe(429) + expect(await response.text()).toContain('revalidated within the last 6 hours') + }) + + it('should revalidate if last revalidation was more than 6 hours ago with basic permissions', async () => { + const request = new NextRequest('https://example.com', { + method: 'POST', + headers: { + Authorization: 'Bearer basic_key', + }, + body: JSON.stringify({ tags: ['tag1'] }), + }) + + vi.mocked(headers).mockReturnValue(new Headers(request.headers)) + + const sevenHoursAgo = new Date(mockDate.getTime() - 7 * 60 * 60 * 1000) + mockSupabaseClient.rpc.mockResolvedValue({ + data: [{ created_at: sevenHoursAgo.toISOString() }], + }) + + const response = await _handleRevalidateRequest(request) + expect(response.status).toBe(204) + expect(revalidateTag).toHaveBeenCalledWith('tag1') + }) + + it('should revalidate regardless of last revalidation time with override permissions', async () => { + const request = new NextRequest('https://example.com', { + method: 'POST', + headers: { + Authorization: 'Bearer override_key', + }, + body: JSON.stringify({ tags: ['tag1'] }), + }) + + vi.mocked(headers).mockReturnValue(new Headers(request.headers)) + + const oneHourAgo = new Date(mockDate.getTime() - 1 * 60 * 60 * 1000) + mockSupabaseClient.rpc.mockResolvedValue({ + data: [{ created_at: oneHourAgo.toISOString() }], + }) + + const response = await _handleRevalidateRequest(request) + expect(response.status).toBe(204) + expect(revalidateTag).toHaveBeenCalledWith('tag1') + }) +}) diff --git a/apps/docs/app/api/revalidate/route.ts b/apps/docs/app/api/revalidate/route.ts new file mode 100644 index 0000000000..5d2a136239 --- /dev/null +++ b/apps/docs/app/api/revalidate/route.ts @@ -0,0 +1,116 @@ +import { createClient } from '@supabase/supabase-js' +import { revalidateTag } from 'next/cache' +import { headers } from 'next/headers' +import { type NextRequest } from 'next/server' +import { z } from 'zod' + +import { type Database } from 'common' + +enum AuthorizationLevel { + Unauthorized, + Basic, + Override, +} + +const requestBodySchema = z.object({ + tags: z.array(z.string()), +}) + +export const POST = handleError(_handleRevalidateRequest) + +export async function _handleRevalidateRequest(request: NextRequest) { + const requestHeaders = headers() + const authorization = requestHeaders.get('Authorization') + if (!authorization) { + return new Response('Missing Authorization header', { status: 401 }) + } + + const basicKeys = process.env.DOCS_REVALIDATION_KEYS?.split(/\s*,\s*/) ?? [] + const overrideKeys = process.env.DOCS_REVALIDATION_OVERRIDE_KEYS?.split(/\s*,\s*/) ?? [] + if (basicKeys.length === 0 && overrideKeys.length === 0) { + console.error('No keys configured for revalidation') + return new Response('Internal server error', { + status: 500, + }) + } + + let authorizationLevel = AuthorizationLevel.Unauthorized + + const token = authorization.replace(/^Bearer\s+/, '') + if (overrideKeys.includes(token)) { + authorizationLevel = AuthorizationLevel.Override + } else if (basicKeys.includes(token)) { + authorizationLevel = AuthorizationLevel.Basic + } + + if (authorizationLevel === AuthorizationLevel.Unauthorized) { + return new Response('Invalid Authorization header', { status: 401 }) + } + + const result = requestBodySchema.safeParse(await request.json()) + if (!result.success) { + return new Response( + 'Malformed request body: should be a JSON object with a "tags" array of strings.', + { status: 400 } + ) + } + + const supabaseAdmin = createClient( + process.env.NEXT_PUBLIC_SUPABASE_URL!, + process.env.SUPABASE_SECRET_KEY! + ) + + if (authorizationLevel === AuthorizationLevel.Basic) { + const { data: lastRevalidation, error } = await supabaseAdmin.rpc( + 'get_last_revalidation_for_tags', + { + tags: result.data.tags, + } + ) + if (error) { + console.error(error) + return new Response('Internal server error', { status: 500 }) + } + + const sixHoursAgo = new Date() + sixHoursAgo.setHours(sixHoursAgo.getHours() - 6) + if (lastRevalidation?.some((revalidation) => new Date(revalidation.created_at) > sixHoursAgo)) { + return new Response( + 'Your request includes a tag that has been revalidated within the last 6 hours. You can override this limit by authenticating with Override permissions.', + { + status: 429, + } + ) + } + } + + const { error } = await supabaseAdmin + .from('validation_history') + .insert(result.data.tags.map((tag) => ({ tag }))) + if (error) { + console.error('Failed to update revalidation table: %o', error) + } + + result.data.tags.forEach((tag) => { + revalidateTag(tag) + }) + + return new Response(null, { + status: 204, + headers: { + 'Cache-Control': 'no-cache', + }, + }) +} + +function handleError(handleRequest: (request: NextRequest) => Promise) { + return async function (request: NextRequest) { + try { + const response = await handleRequest(request) + return response + } catch (error) { + console.error(error) + return new Response('Internal server error', { status: 500 }) + } + } +} diff --git a/packages/common/database-types.ts b/packages/common/database-types.ts index 49a489ef1b..c13e9a2de8 100644 --- a/packages/common/database-types.ts +++ b/packages/common/database-types.ts @@ -58,6 +58,101 @@ export type Database = { } Relationships: [] } + last_changed: { + Row: { + checksum: string + heading: string + id: number + last_checked: string + last_updated: string + parent_page: string + } + Insert: { + checksum: string + heading: string + id?: never + last_checked?: string + last_updated?: string + parent_page: string + } + Update: { + checksum?: string + heading?: string + id?: never + last_checked?: string + last_updated?: string + parent_page?: string + } + Relationships: [] + } + launch_weeks: { + Row: { + created_at: string + end_date: string | null + id: string + start_date: string | null + } + Insert: { + created_at?: string + end_date?: string | null + id: string + start_date?: string | null + } + Update: { + created_at?: string + end_date?: string | null + id?: string + start_date?: string | null + } + Relationships: [] + } + meetups: { + Row: { + country: string | null + created_at: string + display_info: string | null + id: string + is_live: boolean + is_published: boolean + launch_week: string + link: string | null + start_at: string | null + title: string | null + } + Insert: { + country?: string | null + created_at?: string + display_info?: string | null + id?: string + is_live?: boolean + is_published?: boolean + launch_week: string + link?: string | null + start_at?: string | null + title?: string | null + } + Update: { + country?: string | null + created_at?: string + display_info?: string | null + id?: string + is_live?: boolean + is_published?: boolean + launch_week?: string + link?: string | null + start_at?: string | null + title?: string | null + } + Relationships: [ + { + foreignKeyName: "meetups_launch_week_fkey" + columns: ["launch_week"] + isOneToOne: false + referencedRelation: "launch_weeks" + referencedColumns: ["id"] + }, + ] + } page: { Row: { checksum: string | null @@ -141,11 +236,175 @@ export type Database = { }, ] } + tickets: { + Row: { + company: string | null + created_at: string + email: string | null + game_won_at: string | null + id: string + launch_week: string + location: string | null + metadata: Json | null + name: string | null + referred_by: string | null + role: string | null + shared_on_linkedin: string | null + shared_on_twitter: string | null + ticket_number: number + user_id: string + username: string | null + } + Insert: { + company?: string | null + created_at?: string + email?: string | null + game_won_at?: string | null + id?: string + launch_week: string + location?: string | null + metadata?: Json | null + name?: string | null + referred_by?: string | null + role?: string | null + shared_on_linkedin?: string | null + shared_on_twitter?: string | null + ticket_number?: number + user_id: string + username?: string | null + } + Update: { + company?: string | null + created_at?: string + email?: string | null + game_won_at?: string | null + id?: string + launch_week?: string + location?: string | null + metadata?: Json | null + name?: string | null + referred_by?: string | null + role?: string | null + shared_on_linkedin?: string | null + shared_on_twitter?: string | null + ticket_number?: number + user_id?: string + username?: string | null + } + Relationships: [ + { + foreignKeyName: "public_tickets_id_fkey" + columns: ["user_id"] + isOneToOne: false + referencedRelation: "users" + referencedColumns: ["id"] + }, + { + foreignKeyName: "tickets_launch_week_fkey" + columns: ["launch_week"] + isOneToOne: false + referencedRelation: "launch_weeks" + referencedColumns: ["id"] + }, + { + foreignKeyName: "tickets_user_id_fkey" + columns: ["user_id"] + isOneToOne: false + referencedRelation: "users" + referencedColumns: ["id"] + }, + ] + } + troubleshooting_entries: { + Row: { + api: Json | null + date_created: string + date_updated: string + errors: Json[] | null + github_url: string + id: string + keywords: string[] | null + title: string + topics: string[] + } + Insert: { + api?: Json | null + date_created?: string + date_updated?: string + errors?: Json[] | null + github_url: string + id?: string + keywords?: string[] | null + title: string + topics: string[] + } + Update: { + api?: Json | null + date_created?: string + date_updated?: string + errors?: Json[] | null + github_url?: string + id?: string + keywords?: string[] | null + title?: string + topics?: string[] + } + Relationships: [] + } + validation_history: { + Row: { + created_at: string + id: number + tag: string + } + Insert: { + created_at?: string + id?: never + tag: string + } + Update: { + created_at?: string + id?: never + tag?: string + } + Relationships: [] + } } Views: { - [_ in never]: never + tickets_view: { + Row: { + company: string | null + created_at: string | null + id: string | null + launch_week: string | null + location: string | null + metadata: Json | null + name: string | null + platinum: boolean | null + referrals: number | null + role: string | null + secret: boolean | null + shared_on_linkedin: string | null + shared_on_twitter: string | null + ticket_number: number | null + username: string | null + } + Relationships: [ + { + foreignKeyName: "tickets_launch_week_fkey" + columns: ["launch_week"] + isOneToOne: false + referencedRelation: "launch_weeks" + referencedColumns: ["id"] + }, + ] + } } Functions: { + cleanup_last_changed_pages: { + Args: Record + Returns: number + } docs_search_embeddings: { Args: { embedding: string @@ -175,15 +434,13 @@ export type Database = { description: string }[] } - get_page_parents: { + get_last_revalidation_for_tags: { Args: { - page_id: number + tags: string[] } Returns: { - id: number - parent_page_id: number - path: string - meta: Json + tag: string + created_at: string }[] } hnswhandler: { @@ -207,22 +464,6 @@ export type Database = { } Returns: unknown } - match_page_sections: { - Args: { - embedding: string - match_threshold: number - match_count: number - min_content_length: number - } - Returns: { - id: number - page_id: number - slug: string - heading: string - content: string - similarity: number - }[] - } match_page_sections_v2: { Args: { embedding: string @@ -240,6 +481,16 @@ export type Database = { token_count: number | null }[] } + update_last_changed_checksum: { + Args: { + new_parent_page: string + new_heading: string + new_checksum: string + git_update_time: string + check_time: string + } + Returns: string + } vector_avg: { Args: { "": number[] diff --git a/packages/common/package.json b/packages/common/package.json index 7fe6267e31..dc69183069 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -5,6 +5,7 @@ "types": "./index.tsx", "license": "MIT", "scripts": { + "gen:types": "supabase gen types typescript --local >| ./database-types.ts", "typecheck_CURRENTLY_IGNORED": "tsc --noEmit" }, "dependencies": { diff --git a/supabase/migrations/20240918220938_validation_history.sql b/supabase/migrations/20240918220938_validation_history.sql new file mode 100644 index 0000000000..94c78dc9ac --- /dev/null +++ b/supabase/migrations/20240918220938_validation_history.sql @@ -0,0 +1,24 @@ +create table validation_history ( + id bigint generated always as identity primary key, + tag text not null, + created_at timestamp with time zone not null default now() +); + +create index validation_history_tag_created_at_idx on validation_history (tag, created_at desc); + +alter table validation_history enable row level security; + +create or replace function get_last_revalidation_for_tags(tags text[]) +returns table ( + tag text, + created_at timestamp with time zone +) +language sql +as $$ + select + tag, + max(created_at) as created_at + from validation_history + where tag = any(tags) + group by tag; +$$; diff --git a/turbo.json b/turbo.json index 45bd87f524..8ee2d47c9e 100644 --- a/turbo.json +++ b/turbo.json @@ -13,7 +13,15 @@ }, "docs#build": { "dependsOn": ["^build"], - "env": ["ANALYZE", "NEXT_PUBLIC_*", "NODE_ENV", "OPENAI_API_KEY"], + "env": [ + "ANALYZE", + "DOCS_REVALIDATION_KEYS", + "DOCS_REVALIDATION_OVERRIDE_KEYS", + "NEXT_PUBLIC_*", + "NODE_ENV", + "OPENAI_API_KEY", + "SUPABASE_SECRET_KEY" + ], "outputs": [".next/**", "!.next/cache/**"] }, "studio#build": {