Prerendering for Next.js: Setup Guide

1# Find all client component files
2grep -r "'use client'" ./app --include="*.tsx" -l
3
4# Check which routes use client-side data fetching
5grep -r "useEffect\|useSWR\|useQuery" ./app --include="*.tsx" -l

1// middleware.ts
2import { NextRequest, NextResponse } from 'next/server'
3
4const PRERENDERING_SERVICE_URL = process.env.PRERENDERING_SERVICE_URL ?? 'https://ostr.io'
5const PRERENDERING_TOKEN = process.env.PRERENDERING_API_TOKEN ?? ''
6
7// Bot User-Agent patterns — keep updated as new AI crawlers launch
8const BOT_UA_REGEX =
9  /googlebot|bingbot|yandexbot|duckduckbot|slurp|baiduspider|sogou|exabot|facebot|ia_archiver|gptbot|claudebot|applebot|anthropic-ai|perplexitybot|bytespider/i
10
11// Routes to exclude from prerendering
12const EXCLUDED_PATHS = [
13  /^\/_next\//,       // Next.js internal assets
14  /^\/api\//,         // API routes
15  /^\/static\//,      // Static files
16  /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot)$/i, // Assets
17]
18
19function isBot(userAgent: string): boolean {
20  return BOT_UA_REGEX.test(userAgent)
21}
22
23function isExcluded(pathname: string): boolean {
24  return EXCLUDED_PATHS.some((pattern) => pattern.test(pathname))
25}
26
27export function middleware(req: NextRequest) {
28  const ua = req.headers.get('user-agent') ?? ''
29  const { pathname } = req.nextUrl
30
31  // Skip non-bot traffic and excluded paths
32  if (!isBot(ua) || isExcluded(pathname)) {
33    return NextResponse.next()
34  }
35
36  // Build the prerendering request URL
37  const targetUrl = `${PRERENDERING_SERVICE_URL}/render/${req.url}`
38
39  // Forward to prerendering service with auth header
40  return NextResponse.rewrite(targetUrl, {
41    headers: {
42      'X-Prerender-Token': PRERENDERING_TOKEN,
43      'X-Forwarded-For': req.headers.get('x-forwarded-for') ?? req.ip ?? '',
44    },
45  })
46}
47
48export const config = {
49  matcher: [
50    // Match all routes except Next.js internals and static assets
51    '/((?!_next/static|_next/image|favicon.ico).*)',
52  ],
53}

1# /etc/nginx/conf.d/prerendering.conf
2
3mapmap $http_user_agent $is_bot {
4  default 0;
5  "~*googlebot"       1;
6  "~*bingbot"         1;
7  "~*yandexbot"       1;
8  "~*gptbot"          1;
9  "~*claudebot"       1;
10  "~*applebot"        1;
11  "~*anthropic-ai"    1;
12  "~*perplexitybot"   1;
13}
14
15serverserver {
16  listenlisten 443 ssl;
17  server_nameserver_name yourdomain.com;
18
19  locationlocation / {
20    # Route bots to prerendering service
21    ifif ($is_bot) {
22      proxy_passproxy_pass https://ostr.io/render/$scheme://$host$request_uri;
23      proxy_set_header X-Prerender-Token $PRERENDERING_TOKEN;
24      proxy_set_header X-Forwarded-For $remote_addr;
25      breakbreak;
26    }
27
28    # Route users to Next.js app
29    proxy_passproxy_pass http://localhost:3000;
30    proxy_set_header Host $host;
31    proxy_set_header X-Real-IP $remote_addr;
32  }
33}

1# Cloudflare API bulk allowlist
2curl -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/firewall/access_rules/rules" \
3  -H "Authorization: Bearer $CF_API_TOKEN" \
4  -H "Content-Type: application/json" \
5  -d '{"mode":"whitelist","configuration":{"target":"ip_range","value":"<ostr.io CIDR>"},"notes":"ostr.io prerendering"}'

1# Terraform: AWS WAF IP set for ostr.io
2resource "aws_wafv2_ip_set" "ostrio" {
3  name               = "ostr-io-prerendering"
4  scope              = "CLOUDFRONT"
5  ip_address_version = "IPV4"
6  addresses          = [/* ostr.io IP ranges */]
7}
8
9resource "aws_wafv2_web_acl_rule" "allow_ostrio" {
10  name     = "AllowOstrio"
11  priority = 1
12  action { allow {} }
13  statement {
14    ip_set_reference_statement {
15      arn = aws_wafv2_ip_set.ostrio.arn
16    }
17  }
18  visibility_config {
19    cloudwatch_metrics_enabled = true
20    metric_name                = "AllowOstrio"
21    sampled_requests_enabled   = true
22  }
23}

1// ostr.io TTL configuration via API
2const ttlConfig = {
3  patterns: [
4    { match: '/blog/**', ttl: 3600 },          // Blog posts: 1 hour
5    { match: '/product/**', ttl: 900 },         // Products: 15 min (pricing)
6    { match: '/category/**', ttl: 1800 },       // Categories: 30 min
7    { match: '/docs/**', ttl: 86400 },          // Docs: 24 hours
8    { match: '/', ttl: 3600 },                  // Homepage: 1 hour
9  ]
10}

1// lib/prerendering.ts
2export async function warmSnapshot(url: string, priority: 'high' | 'normal' = 'normal') {
3  if (!process.env.OSTRIO_API_KEY) return
4
5  try {
6    await fetch('https://ostr.io/api/cache-warm', {
7      method: 'POST',
8      headers: {
9        Authorization: `Bearer ${process.env.OSTRIO_API_KEY}`,
10        'Content-Type': 'application/json',
11      },
12      body: JSON.stringify({ url, priority }),
13    })
14  } catch {
15    // Non-critical — log but don't throw
16    console.warn(`Cache warm failed for ${url}`)
17  }
18}
19
20// In your CMS webhook handler or Server Action
21import { revalidatePath } from 'next/cache'
22import { warmSnapshot } from '@/lib/prerendering'
23
24export async function publishContent(slug: string) {
25  const url = `https://yourdomain.com/blog/${slug}`
26
27  // Revalidate ISR cache
28  revalidatePath(`/blog/${slug}`)
29
30  // Warm prerendering snapshot
31  await warmSnapshot(url, 'high')
32}

1# Simulate Googlebot request
2curl -A "Googlebot" https://yourdomain.com/blog/your-post -I
3
4# Expected: X-Prerender-Status: hit or X-Prerender-Status: rendered header
5# If missing: middleware is not routing bot traffic correctly

1# Simulate user request (no bot UA)
2curl https://yourdomain.com/blog/your-post -I
3
4# Expected: no prerender headers, normal Next.js response

1# Check JSON-LD in prerendered output
2curl -A "Googlebot" https://yourdomain.com/product/your-product | grep -o 'application/ld+json' | wc -l
3# Should return 1 or more