vercel__next.js/test/e2e/prerender-crawler.test.ts

import { nextTestSetup } from 'e2e-utils'
import { renderViaHTTP } from 'next-test-utils'

describe('Prerender crawler handling', () => {
  const { next } = nextTestSetup({
    files: {
      'pages/index.js': `
        export default function Page() { 
          return <p>index page</p>
        } 
      `,
      'pages/blog/[slug].js': `
        import {useRouter} from 'next/router'
        
        export default function Page({ slug }) { 
          const router = useRouter()
          
          if (router.isFallback) {
            return 'Loading...'
          }
          
          return (
            <>
              <p id='page'>slug page</p>
              <p id='slug'>{slug}</p>
            </>
          )
        } 
        
        export async function getStaticProps({ params }) {
          return {
            props: {
              slug: params.slug
            }
          }
        } 
        
        export async function getStaticPaths() {
          return {
            paths: ['/blog/first'],
            fallback: true
          }
        }
      `,
    },
  })

  it('should return prerendered page for correctly', async () => {
    const html = await renderViaHTTP(next.url, '/blog/first')
    expect(html).not.toContain('Loading...')
    expect(html).toContain('first')
    expect(html).toContain('slug page')
  })

  it('should return fallback for non-crawler correctly', async () => {
    const html = await renderViaHTTP(next.url, '/blog/test-1', undefined, {
      headers: {
        'user-agent':
          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36',
      },
    })
    expect(html).toContain('Loading...')
  })

  it('should block for crawler correctly', async () => {
    const userAgents = [
      'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
      'Mozilla/5.0 (compatible; Bingbot/2.0; +http://www.bing.com/bingbot.htm)',
      'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)',
      'DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)',
      'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
      'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)',
      'facebookexternalhit/1.0 (+http://www.facebook.com/externalhit_uatext.php)',
      'ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)',
    ]

    for (const userAgent of userAgents) {
      console.log('checking', { userAgent })
      const uniqueSlug = `test-${Date.now()}${Math.random()}`

      const html = await renderViaHTTP(
        next.url,
        `/blog/${uniqueSlug}`,
        undefined,
        {
          headers: {
            'user-agent': userAgent,
          },
        }
      )
      expect(html).not.toContain('Loading...')
      expect(html).toContain(uniqueSlug)
      expect(html).toContain('slug page')
    }
  })
})