From b62a4af619a9af4ecb31d015ed1ad68e1c673a93 Mon Sep 17 00:00:00 2001 From: Alaister Young Date: Wed, 5 Feb 2025 13:50:36 +0800 Subject: [PATCH] chore: static assets cdn (#33304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: static assets cdn * fix build command for vercel * try a different aws setup script * use a specific aws cli version for r2 compat * clean up local static files * use more env vars * Update upload-static-assets.sh * Update turbo.json * Update turbo.json * moar * Update upload-static-assets.sh * Update upload-static-assets.sh * hard disable * Update upload-static-assets.sh * Update next.config.js * Update upload-static-assets.sh * add supabase assets url to image src urls * add site name to turbo.json env vars --------- Co-authored-by: Kevin Grüneberg --- apps/studio/next.config.js | 27 ++++++++-- apps/studio/vercel.json | 3 ++ scripts/upload-static-assets.sh | 94 +++++++++++++++++++++++++++++++++ turbo.json | 7 ++- 4 files changed, 126 insertions(+), 5 deletions(-) create mode 100644 apps/studio/vercel.json create mode 100755 scripts/upload-static-assets.sh diff --git a/apps/studio/next.config.js b/apps/studio/next.config.js index 6b9fb11e30..c20e986091 100644 --- a/apps/studio/next.config.js +++ b/apps/studio/next.config.js @@ -51,16 +51,20 @@ const GOOGLE_USER_AVATAR_URL = 'https://lh3.googleusercontent.com' const VERCEL_LIVE_URL = 'https://vercel.live' const SENTRY_URL = 'https://*.ingest.sentry.io https://*.ingest.us.sentry.io https://*.ingest.de.sentry.io' +const SUPABASE_ASSETS_URL = + process.env.NEXT_PUBLIC_ENVIRONMENT === 'staging' + ? 'https://frontend-assets.supabase.green' + : 'https://frontend-assets.supabase.com' // used by vercel live preview const PUSHER_URL = 'https://*.pusher.com' const PUSHER_URL_WS = 'wss://*.pusher.com' -const DEFAULT_SRC_URLS = `${API_URL} ${SUPABASE_URL} ${GOTRUE_URL} ${SUPABASE_LOCAL_PROJECTS_URL_WS} ${SUPABASE_PROJECTS_URL} ${SUPABASE_PROJECTS_URL_WS} ${HCAPTCHA_SUBDOMAINS_URL} ${CONFIGCAT_URL} ${STRIPE_SUBDOMAINS_URL} ${STRIPE_NETWORK_URL} ${CLOUDFLARE_URL} ${ONE_ONE_ONE_ONE_URL} ${VERCEL_INSIGHTS_URL} ${GITHUB_API_URL} ${GITHUB_USER_CONTENT_URL}` -const SCRIPT_SRC_URLS = `${CLOUDFLARE_CDN_URL} ${HCAPTCHA_JS_URL} ${STRIPE_JS_URL}` +const DEFAULT_SRC_URLS = `${API_URL} ${SUPABASE_URL} ${GOTRUE_URL} ${SUPABASE_LOCAL_PROJECTS_URL_WS} ${SUPABASE_PROJECTS_URL} ${SUPABASE_PROJECTS_URL_WS} ${HCAPTCHA_SUBDOMAINS_URL} ${CONFIGCAT_URL} ${STRIPE_SUBDOMAINS_URL} ${STRIPE_NETWORK_URL} ${CLOUDFLARE_URL} ${ONE_ONE_ONE_ONE_URL} ${VERCEL_INSIGHTS_URL} ${GITHUB_API_URL} ${GITHUB_USER_CONTENT_URL} ${SUPABASE_ASSETS_URL}` +const SCRIPT_SRC_URLS = `${CLOUDFLARE_CDN_URL} ${HCAPTCHA_JS_URL} ${STRIPE_JS_URL} ${SUPABASE_ASSETS_URL}` const FRAME_SRC_URLS = `${HCAPTCHA_ASSET_URL} ${STRIPE_JS_URL}` -const IMG_SRC_URLS = `${SUPABASE_URL} ${SUPABASE_COM_URL} ${SUPABASE_PROJECTS_URL} ${GITHUB_USER_AVATAR_URL} ${GOOGLE_USER_AVATAR_URL}` -const STYLE_SRC_URLS = `${CLOUDFLARE_CDN_URL}` +const IMG_SRC_URLS = `${SUPABASE_URL} ${SUPABASE_COM_URL} ${SUPABASE_PROJECTS_URL} ${GITHUB_USER_AVATAR_URL} ${GOOGLE_USER_AVATAR_URL} ${SUPABASE_ASSETS_URL}` +const STYLE_SRC_URLS = `${CLOUDFLARE_CDN_URL} ${SUPABASE_ASSETS_URL}` const FONT_SRC_URLS = `${CLOUDFLARE_CDN_URL}` const csp = [ @@ -96,11 +100,26 @@ const csp = [ : []), ].join(' ') +function getAssetPrefix() { + // If not force enabled, but not production env, disable CDN + if (process.env.FORCE_ASSET_CDN !== '1' && process.env.VERCEL_ENV !== 'production') { + return undefined + } + + // Force disable CDN + if (process.env.FORCE_ASSET_CDN === '-1') { + return undefined + } + + return `${SUPABASE_ASSETS_URL}/${process.env.SITE_NAME}/${process.env.VERCEL_GIT_COMMIT_SHA.substring(0, 12)}` +} + /** * @type {import('next').NextConfig} */ const nextConfig = { basePath: process.env.NEXT_PUBLIC_BASE_PATH, + assetPrefix: getAssetPrefix(), output: 'standalone', experimental: { webpackBuildWorker: true, diff --git a/apps/studio/vercel.json b/apps/studio/vercel.json new file mode 100644 index 0000000000..e2a5e6299e --- /dev/null +++ b/apps/studio/vercel.json @@ -0,0 +1,3 @@ +{ + "buildCommand": "next build && ./../../scripts/upload-static-assets.sh" +} diff --git a/scripts/upload-static-assets.sh b/scripts/upload-static-assets.sh new file mode 100755 index 0000000000..f3bfc48a2b --- /dev/null +++ b/scripts/upload-static-assets.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +####### + +# This script is used to upload static build assets (JS, CSS, ...) and public static files (public folder) to a CDN. +# We're using Cloudflare R2 as CDN. +# By using a CDN, we can serve static assets extremely fast while saving big time on egress costs. +# An alternative is proxying via CF, but that comes with Orange-To-Orange issues (Cloudflare having issues with Cloudflare) and increased latency as there is a double TLS termination. +# The script is only supposed to run on production deployments and is not run on any previews. + +# By using a dynamic path including the env, app and commit hash, we can ensure that there are no conflicts. +# Static assets from previous deployments stick around for a while to ensure there are no "downtimes". + +# Advantages of the CDN approach we're using: + +# Get rid of egress costs for static assets across our apps on Vercel +# Disable CF proxying and get around these odd timeouts issues +# Save ~20ms or so for asset requests, as there is no additional CF proxying and we avoid terminating SSL twice +# Always hits the CDN, gonna be super quick +# Does not run on local or preview environments, only on staging/prod deployments +# There are no other disadvantages - you don't have to consider it when developing locally, previews still work, everything on Vercel works as we're used to + +####### + +# If asset CDN is specifically disabled (i.e. studio self-hosted), we skip +if [[ "$FORCE_ASSET_CDN" == "-1" ]]; then + echo "Skipping asset upload. Set FORCE_ASSET_CDN=1 or VERCEL_ENV=production to execute." + exit 0 +fi + +# Check for force env var or production environment +if [[ "$FORCE_ASSET_CDN" != "1" ]] && [[ "$VERCEL_ENV" != "production" ]]; then + echo "Skipping asset upload. Set FORCE_ASSET_CDN=1 or VERCEL_ENV=production to execute." + exit 0 +fi + +# Set the cdnBucket variable based on NEXT_PUBLIC_ENVIRONMENT +if [[ "$NEXT_PUBLIC_ENVIRONMENT" == "staging" ]]; then + BUCKET_NAME="frontend-assets-staging" +else + BUCKET_NAME="frontend-assets-prod" +fi + +STATIC_DIR=".next/static" +PUBLIC_DIR="public" + +# Colors for output +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Install AWS CLI if not present +if ! command -v aws &> /dev/null; then + echo -e "${YELLOW}Setting up AWS CLI...${NC}" + curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip" + unzip -q awscliv2.zip + export PATH=$PWD/aws/dist:$PATH + rm awscliv2.zip +fi + +# Check if directory exists +if [ ! -d "$STATIC_DIR" ]; then + echo -e "${YELLOW}Directory $STATIC_DIR not found!${NC}" + echo "Make sure you're running this script from your Next.js project root." + exit 1 +fi + +# Upload files with cache configuration and custom endpoint +echo -e "${YELLOW}Uploading static files to R2...${NC}" +aws s3 sync "$STATIC_DIR" "s3://$BUCKET_NAME/$SITE_NAME/${VERCEL_GIT_COMMIT_SHA:0:12}/_next/static" \ + --endpoint-url "$ASSET_CDN_S3_ENDPOINT" \ + --cache-control "public,max-age=604800,immutable" \ + --region auto \ + --only-show-errors + +# Some public files may be referenced through CSS (relative path) and therefore they would be requested via the CDN url +# To ensure we don't run into some nasty debugging issues, we upload the public files to the CDN as well +echo -e "${YELLOW}Uploading public files to R2...${NC}" +aws s3 sync "$PUBLIC_DIR" "s3://$BUCKET_NAME/$SITE_NAME/${VERCEL_GIT_COMMIT_SHA:0:12}" \ + --endpoint-url "$ASSET_CDN_S3_ENDPOINT" \ + --cache-control "public,max-age=604800,immutable" \ + --region auto \ + --only-show-errors + +if [ $? -eq 0 ]; then + echo -e "${GREEN}Upload completed successfully!${NC}" + + # Clean up local static files so we prevent a double upload + echo -e "${YELLOW}Cleaning up local static files...${NC}" + rm -rf "$STATIC_DIR"/* + echo -e "${GREEN}Local static files cleaned up${NC}" + + # We still keep the public dir, as Next.js does not officially support serving the public files via CDN +fi \ No newline at end of file diff --git a/turbo.json b/turbo.json index cf669cffc8..9073806fcf 100644 --- a/turbo.json +++ b/turbo.json @@ -65,7 +65,12 @@ "LOGFLARE_API_KEY", "SENTRY_ORG", "SENTRY_PROJECT", - "SENTRY_AUTH_TOKEN" + "SENTRY_AUTH_TOKEN", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "FORCE_ASSET_CDN", + "ASSET_CDN_S3_ENDPOINT", + "SITE_NAME" ], "outputs": [".next/**", "!.next/cache/**"] },