fix,feat(sitemap): add all pages + priority (#29083)

Pages were missing from the sitemap after the reorganization. Added them
back in.

Also added an optional priority that can be specified by including
`sitemapPriority` in content metadata. Used on the deprecated Auth
Helpers pages to see if we can tweak the relative SEO ranking for them.

apps/docs/content/guides/auth/auth-helpers/auth-ui.mdx

@@ -2,6 +2,7 @@
 id: 'auth-ui'
 title: 'Auth UI'
 description: 'A prebuilt, customizable React component for authenticating users.'
+sitemapPriority: 0.5
 ---
 
 <Admonition type="warning">

apps/docs/content/guides/auth/auth-helpers/flutter-auth-ui.mdx

@@ -2,6 +2,7 @@
 id: 'flutter-auth-ui'
 title: 'Flutter Auth UI'
 description: 'Prebuilt, customizable Flutter widgets for authenticating users.'
+sitemapPriority: 0.5
 ---
 
 Flutter Auth UI is a Flutter package containing pre-built widgets for authenticating users.

apps/docs/content/guides/auth/auth-helpers/nextjs-pages.mdx

@@ -3,6 +3,7 @@ id: 'nextjs-pages'
 title: 'Supabase Auth with Next.js Pages Directory'
 description: 'Authentication helpers for Next.js API routes, middleware, and SSR in the Pages Directory.'
 sidebar_label: 'Next.js (pages)'
+sitemapPriority: 0.5
 ---
 
 <Admonition type="caution">

apps/docs/content/guides/auth/auth-helpers/nextjs.mdx

@@ -3,6 +3,7 @@ id: 'nextjs'
 title: 'Supabase Auth with the Next.js App Router'
 description: 'Authentication and Authorization helpers for creating an authenticated Supabase client with the Next.js 13 App Router.'
 sidebar_label: 'Next.js'
+sitemapPriority: 0.5
 ---
 
 <Admonition type="caution">

apps/docs/content/guides/auth/auth-helpers/remix.mdx

@@ -3,6 +3,7 @@ id: 'remix'
 title: 'Supabase Auth with Remix'
 description: 'Authentication helpers for loaders and actions in Remix.'
 sidebar_label: 'Remix'
+sitemapPriority: 0.5
 ---
 
 <Admonition type="caution">

apps/docs/content/guides/auth/auth-helpers/sveltekit.mdx

@@ -3,6 +3,7 @@ id: 'sveltekit'
 title: 'Supabase Auth with SvelteKit'
 description: 'Convenience helpers for implementing user authentication in SvelteKit.'
 sidebar_label: 'SvelteKit'
+sitemapPriority: 0.5
 ---
 
 <Admonition type="caution">

apps/docs/internals/files/api.mjs

@@ -1,13 +0,0 @@
-import sections from '../../spec/common-api-sections.json' assert { type: 'json' }
-import { flattenSections } from '../helpers.mjs'
-
-const flatSections = flattenSections(sections)
-
-export function generateAPIPages() {
-  let apiPages = []
-
-  flatSections.map((section) => {
-    apiPages.push(`reference/api/${section.slug}`)
-  })
-  return apiPages
-}

apps/docs/internals/files/api.ts

@@ -0,0 +1,17 @@
+import sections from '../../spec/common-api-sections.json'
+import { flattenSections } from '../helpers'
+
+const flatSections = flattenSections(sections)
+
+/**
+ * Generates the API page links for the management API reference.
+ * @returns {Array<{link: string}>} - An array of API page links.
+ */
+export function generateAPIPages() {
+  let apiPages = []
+
+  flatSections.map((section) => {
+    apiPages.push({ link: `reference/api/${section.slug}` })
+  })
+  return apiPages
+}

apps/docs/internals/files/cli.ts

@@ -1,18 +1,23 @@
 import fs from 'fs'
 import yaml from 'js-yaml'
-import cliCommonSections from '../../spec/common-cli-sections.json' assert { type: 'json' }
-import { flattenSections } from '../helpers.mjs'
+
+import cliCommonSections from '../../spec/common-cli-sections.json'
+import { flattenSections } from '../helpers'
 
 const flatCLISections = flattenSections(cliCommonSections)
 
-const cliSpec = yaml.load(fs.readFileSync(`spec/cli_v1_commands.yaml`, 'utf8'))
+const cliSpec = yaml.load(fs.readFileSync(`spec/cli_v1_commands.yaml`, 'utf8')) as any
 
+/**
+ * Generates the CLI page links for the CLI reference.
+ * @returns {Array<{link: string}>} - An array of CLI page links.
+ */
 export function generateCLIPages() {
   let cliPages = []
 
   cliSpec.commands.map((section) => {
     const slug = flatCLISections.find((item) => item.id === section.id)?.slug
-    if (slug) cliPages.push(`reference/cli/${slug}`)
+    if (slug) cliPages.push({ link: `reference/cli/${slug}` })
   })
   return cliPages
 }

apps/docs/internals/files/reference-lib.mjs

@@ -1,28 +0,0 @@
-import fs from 'fs'
-
-import yaml from 'js-yaml'
-import commonLibSections from '../../spec/common-client-libs-sections.json' assert { type: 'json' }
-import { flattenSections } from '../helpers.mjs'
-
-const flatCommonLibSections = flattenSections(commonLibSections)
-
-const clientLibFiles = [
-  { fileName: 'supabase_js_v2', label: 'javascript', version: 'v2', versionSlug: false },
-  { fileName: 'supabase_dart_v2', label: 'dart', version: 'v2', versionSlug: false },
-  { fileName: 'supabase_py_v2', label: 'python', version: 'v2', versionSlug: false },
-  { fileName: 'supabase_csharp_v0', label: 'csharp', version: 'v0', versionSlug: false },
-  { fileName: 'supabase_swift_v1', label: 'swift', version: 'v1', versionSlug: false },
-  { fileName: 'supabase_kt_v2', label: 'kotlin', version: 'v2', versionSlug: false },
-]
-
-export function generateReferencePages() {
-  let refPages = []
-  clientLibFiles.map((file) => {
-    const spec = yaml.load(fs.readFileSync(`spec/${file.fileName}.yml`, 'utf8'))
-    spec.functions.map((fn) => {
-      const slug = flatCommonLibSections.find((item) => item.id === fn.id)?.slug
-      refPages.push(`reference/${file.label}/${file.versionSlug ? file.version + '/' : ''}${slug}`)
-    })
-  })
-  return refPages
-}

apps/docs/internals/files/reference-lib.ts

@@ -0,0 +1,30 @@
+import { REFERENCES } from '../../content/navigation.references'
+import { getFlattenedSections } from '../../features/docs/Reference.generated.singleton'
+
+export async function generateReferencePages() {
+  return (
+    await Promise.all(
+      Object.keys(REFERENCES)
+        .flatMap((key) => {
+          return REFERENCES[key].versions.map((version) => {
+            return {
+              sdkId: key,
+              version: version,
+              isLatestVersion: version === REFERENCES[key].versions[0],
+              libPath: REFERENCES[key].libPath,
+            }
+          })
+        })
+        .map(async ({ sdkId, version, libPath, isLatestVersion }) => {
+          const flattenedSections = await getFlattenedSections(sdkId, version)
+          return (
+            flattenedSections?.map((section) => ({
+              link: isLatestVersion
+                ? `reference/${libPath}/${section.slug}`
+                : `reference/${libPath}/${version}/${section.slug}`,
+            })) ?? []
+          )
+        })
+    )
+  ).flat()
+}

apps/docs/internals/generate-sitemap.mjs

@@ -1,69 +0,0 @@
-/*
- * kudos to leerob from vercel
- * https://leerob.io/blog/nextjs-sitemap-robots
- */
-
-import { writeFileSync } from 'fs'
-import { globby } from 'globby'
-import prettier from 'prettier'
-import { generateCLIPages } from './files/cli.mjs'
-import { generateReferencePages } from './files/reference-lib.mjs'
-import { generateAPIPages } from './files/api.mjs'
-
-const referencePages = generateReferencePages()
-const cliPages = generateCLIPages()
-const apiPages = generateAPIPages()
-
-async function generate() {
-  const prettierConfig = await prettier.resolveConfig('./.prettierrc.js')
-
-  const rawPages = await globby([
-    // guides
-    'docs/*.mdx',
-    'pages/**/*.mdx',
-    '!pages/404.mdx',
-    '!pages/404.tsx',
-    '!pages/ref-pages.mdx',
-  ])
-
-  const guidePages = rawPages.map((x) => {
-    let string = x
-    string = string.replace('pages/', '')
-    string = string.replace('.mdx', '')
-    return string
-  })
-
-  // combine the guidePages with the ref pages
-  const allPages = guidePages.concat(referencePages, cliPages, apiPages)
-
-  const sitemap = `
-    <?xml version="1.0" encoding="UTF-8"?>
-    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-        ${allPages
-          .map((path) => {
-            return `
-              <url>
-                  <loc>${`https://supabase.com/docs/${path}`}</loc>
-                  <changefreq>weekly</changefreq>
-              </url>
-            `
-          })
-          .join('')}
-    </urlset>
-    `
-
-  const formatted = await prettier.format(sitemap, {
-    ...prettierConfig,
-    parser: 'html',
-  })
-
-  const sitemapFilePath = `public/sitemap.xml`
-  console.log(
-    `Total of ${allPages.length} pages in sitemap, located at /apps/docs/${sitemapFilePath}`
-  )
-
-  // eslint-disable-next-line no-sync
-  writeFileSync(sitemapFilePath, formatted)
-}
-
-generate()

apps/docs/internals/generate-sitemap.ts

@@ -0,0 +1,73 @@
+/*
+ * kudos to leerob from vercel
+ * https://leerob.io/blog/nextjs-sitemap-robots
+ */
+
+import fs from 'fs'
+import { globby } from 'globby'
+import matter from 'gray-matter'
+import prettier from 'prettier'
+
+import { generateAPIPages } from './files/api'
+import { generateCLIPages } from './files/cli'
+import { generateReferencePages } from './files/reference-lib'
+
+async function generate() {
+  const apiPages = generateAPIPages()
+  const cliPages = generateCLIPages()
+  const referencePages = await generateReferencePages()
+
+  const contentFiles = await globby(['content/**/!(_)*.mdx'])
+  const contentPages = await Promise.all(
+    contentFiles.map(async (filePath) => {
+      const fileContents = await fs.promises.readFile(filePath, 'utf8')
+      const {
+        data: { sitemapPriority },
+      } = matter(fileContents)
+
+      return {
+        link: filePath.replace(/^content\//, '').replace(/\.mdx$/, ''),
+        priority: sitemapPriority,
+      }
+    })
+  )
+
+  const allPages = (contentPages as Array<{ link: string; priority?: number }>).concat(
+    referencePages,
+    cliPages,
+    apiPages
+  )
+
+  const sitemap = `
+    <?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        ${allPages
+          .map(({ link, priority }) => {
+            return `
+              <url>
+                  <loc>${`https://supabase.com/docs/${link}`}</loc>
+                  <changefreq>weekly</changefreq>
+                  ${priority ? `<priority>${priority}</priority>` : ''}
+              </url>
+            `
+          })
+          .join('')}
+    </urlset>
+    `
+
+  const prettierConfig = await prettier.resolveConfig('./.prettierrc.js')
+  const formatted = await prettier.format(sitemap, {
+    ...prettierConfig,
+    parser: 'html',
+  })
+
+  const sitemapFilePath = `public/sitemap.xml`
+  console.log(
+    `Total of ${allPages.length} pages in sitemap, located at /apps/docs/${sitemapFilePath}`
+  )
+
+  // eslint-disable-next-line no-sync
+  fs.writeFileSync(sitemapFilePath, formatted)
+}
+
+generate()

apps/docs/internals/helpers.ts

@@ -1,6 +1,6 @@
 export function flattenSections(sections) {
-  var a = []
-  for (var i = 0; i < sections.length; i++) {
+  let a = []
+  for (let i = 0; i < sections.length; i++) {
     if (sections[i].id) {
       // only push a section that has an id
       // these are reserved for sidebar subtitles

apps/docs/package.json

@@ -14,14 +14,14 @@
     "typecheck": "tsc --noEmit",
     "test": "vitest --exclude \"**/*.smoke.test.ts\"",
     "test:smoke": "npm run codegen:references && vitest -t \"prod smoke test\"",
-    "build:sitemap": "node ./internals/generate-sitemap.mjs",
+    "build:sitemap": "tsx ./internals/generate-sitemap.ts",
     "embeddings": "tsx scripts/search/generate-embeddings.ts",
     "embeddings:refresh": "npm run embeddings -- --refresh",
     "last-changed": "tsx scripts/last-changed.ts",
     "last-changed:reset": "npm run last-changed -- --reset",
     "codegen:references": "tsx features/docs/Reference.generated.script.ts",
     "codemod:frontmatter": "node ./scripts/codemod/mdx-meta.mjs && prettier --write \"content/**/*.mdx\"",
-    "postbuild": "node ./internals/generate-sitemap.mjs"
+    "postbuild": "npm run build:sitemap"
   },
   "dependencies": {
     "@code-hike/mdx": "^0.9.0",