Repo sync

.env.example

@@ -20,4 +20,9 @@ BUILD_RECORDS_MAX_CONCURRENT=100
 BUILD_RECORDS_MIN_TIME=
 
 # Set to true to enable the /fastly-cache-test route for debugging Fastly headers
-ENABLE_FASTLY_TESTING=
+ENABLE_FASTLY_TESTING=
+
+# Needed to auth for AI search
+CSE_COPILOT_SECRET=
+CSE_COPILOT_ENDPOINT=https://cse-copilot-staging.service.iad.github.net
+

.github/workflows/codeowners-content-strategy.yml → .github/workflows/codeowners-content-systems.yml

@@ -11,14 +11,14 @@ on:
       - 'content/contributing/**.md'
 
 jobs:
-  codeowners-content-strategy:
+  codeowners-content-systems:
     if: ${{ github.repository == 'github/docs-internal' }}
     runs-on: ubuntu-latest
     steps:
       - name: Check out repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-      - name: Add Content Strategy as a reviewer
+      - name: Add Content Systems as a reviewer
         env:
           GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_WRITEORG_PROJECT }}
           PR: ${{ github.event.pull_request.html_url }}
@@ -29,5 +29,5 @@ jobs:
           )
           if ! $has_reviewer
           then
-            gh pr edit $PR --add-reviewer github/docs-content-strategy
+            gh pr edit $PR --add-reviewer github/docs-content-systems
           fi

package.json

@@ -33,7 +33,7 @@
     "delete-orphan-translation-files": "tsx src/workflows/delete-orphan-translation-files.ts",
     "deleted-features-pr-comment": "tsx src/data-directory/scripts/deleted-features-pr-comment.ts",
     "dev": "cross-env npm start",
-    "find-orphaned-assets": "node src/assets/scripts/find-orphaned-assets.js",
+    "find-orphaned-assets": "tsx src/assets/scripts/find-orphaned-assets.ts",
     "find-orphaned-features": "tsx src/data-directory/scripts/find-orphaned-features/index.ts",
     "find-past-built-pr": "tsx src/workflows/find-past-built-pr.ts",
     "find-unused-variables": "tsx src/content-linter/scripts/find-unsed-variables.ts",
@@ -259,7 +259,7 @@
     "express": "4.21.1",
     "express-rate-limit": "7.4.0",
     "fastest-levenshtein": "1.0.16",
-    "file-type": "19.4.1",
+    "file-type": "19.6.0",
     "flat": "^6.0.1",
     "github-slugger": "^2.0.0",
     "glob": "11.0.0",

src/assets/lib/image-density.d.ts

@@ -0,0 +1 @@
+export const IMAGE_DENSITY: Record<string, string>

src/assets/scripts/deleted-assets-pr-comment-1.js → src/assets/scripts/deleted-assets-pr-comment-1.ts

@@ -18,14 +18,19 @@
 // [end-readme]
 
 import { program } from 'commander'
-import main from './deleted-assets-pr-comment.js'
+import main from './deleted-assets-pr-comment'
 
 program
   .description('If applicable, print a snippet of Markdown about deleted assets')
-  .arguments('owner repo base_sha head_sha', 'Simulate what the Actions workflow does')
+  .arguments('owner repo base_sha head_sha')
   .parse(process.argv)
 
-const opts = program.opts()
-const args = program.args
+type MainArgs = {
+  owner: string
+  repo: string
+  baseSHA: string
+  headSHA: string
+}
+const opts = program.opts() as MainArgs
 
-console.log(await main(...args, { ...opts }))
+console.log(await main(opts))

src/assets/scripts/deleted-assets-pr-comment.js → src/assets/scripts/deleted-assets-pr-comment.ts

@@ -13,16 +13,22 @@ if (!GITHUB_TOKEN) {
 // When this file is invoked directly from action as opposed to being imported
 if (import.meta.url.endsWith(process.argv[1])) {
   const owner = context.repo.owner
-  const repo = context.payload.repository.name
-  const baseSHA = context.payload.pull_request.base.sha
-  const headSHA = context.payload.pull_request.head.sha
+  const repo = context.payload.repository?.name || ''
+  const baseSHA = context.payload.pull_request?.base.sha
+  const headSHA = context.payload.pull_request?.head.sha
 
-  const markdown = await main(owner, repo, baseSHA, headSHA)
+  const markdown = await main({ owner, repo, baseSHA, headSHA })
   core.setOutput('markdown', markdown)
 }
 
-async function main(owner, repo, baseSHA, headSHA) {
-  const octokit = github.getOctokit(GITHUB_TOKEN)
+type MainArgs = {
+  owner: string
+  repo: string
+  baseSHA: string
+  headSHA: string
+}
+async function main({ owner, repo, baseSHA, headSHA }: MainArgs) {
+  const octokit = github.getOctokit(GITHUB_TOKEN as string)
   // get the list of file changes from the PR
   const response = await octokit.rest.repos.compareCommitsWithBasehead({
     owner,
@@ -32,6 +38,10 @@ async function main(owner, repo, baseSHA, headSHA) {
 
   const { files } = response.data
 
+  if (!files) {
+    throw new Error('No files found in the PR')
+  }
+
   const oldFilenames = []
   for (const file of files) {
     const { filename, status } = file

src/assets/scripts/find-orphaned-assets.js → src/assets/scripts/find-orphaned-assets.ts

@@ -32,7 +32,7 @@ const EXCEPTIONS = new Set([
   'assets/images/site/apple-touch-icon-76x76.png',
 ])
 
-function isExceptionPath(imagePath) {
+function isExceptionPath(imagePath: string) {
   // We also check for .DS_Store because any macOS user that has opened
   // a folder with images will have this on disk. It won't get added
   // to git anyway thanks to our .DS_Store.
@@ -53,9 +53,15 @@ program
   .option('--exclude-translations', "Don't search in translations/")
   .parse(process.argv)
 
-main(program.opts(), program.args)
+type MainOptions = {
+  json: boolean
+  verbose: boolean
+  exit: boolean
+  excludeTranslations: boolean
+}
+main(program.opts())
 
-async function main(opts) {
+async function main(opts: MainOptions) {
   const { json, verbose, exit, excludeTranslations } = opts
 
   const walkOptions = {
@@ -164,7 +170,7 @@ async function main(opts) {
   }
 }
 
-function getTotalDiskSize(filePaths) {
+function getTotalDiskSize(filePaths: Set<string>) {
   let sum = 0
   for (const filePath of filePaths) {
     sum += fs.statSync(filePath).size

src/assets/scripts/list-image-sizes.js → src/assets/scripts/list-image-sizes.ts

@@ -10,32 +10,26 @@ import { fileURLToPath } from 'url'
 import path from 'path'
 import walk from 'walk-sync'
 import sharp from 'sharp'
-import { chain } from 'lodash-es'
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 
 const imagesPath = path.join(__dirname, '../assets/images')
 const imagesExtensions = ['.jpg', '.jpeg', '.png', '.gif']
 
-const files = chain(walk(imagesPath, { directories: false })).filter((relativePath) => {
+const files = walk(imagesPath, { directories: false }).filter((relativePath) => {
   return imagesExtensions.includes(path.extname(relativePath.toLowerCase()))
 })
-const infos = await Promise.all(
+const images = await Promise.all(
   files.map(async (relativePath) => {
     const fullPath = path.join(imagesPath, relativePath)
     const image = sharp(fullPath)
     const { width, height } = await image.metadata()
-    const size = width * height
+    const size = (width || 0) * (height || 0)
     return { relativePath, width, height, size }
   }),
 )
-const images = files
-  .map((relativePath, i) => {
-    return { relativePath, ...infos[i] }
+images
+  .sort((a, b) => b.size - a.size)
+  .forEach((image) => {
+    const { relativePath, width, height } = image
+    console.log(`${width} x ${height} - ${relativePath}`)
   })
-  .orderBy('size', 'desc')
-  .value()
-
-images.forEach((image) => {
-  const { relativePath, width, height } = image
-  console.log(`${width} x ${height} - ${relativePath}`)
-})

src/assets/scripts/validate-asset-images.js → src/assets/scripts/validate-asset-images.ts

@@ -19,6 +19,7 @@ import path from 'path'
 import { program } from 'commander'
 import chalk from 'chalk'
 import cheerio from 'cheerio'
+// @ts-ignore see https://github.com/sindresorhus/file-type/issues/652
 import { fileTypeFromFile } from 'file-type'
 import walk from 'walk-sync'
 import isSVG from 'is-svg'
@@ -43,7 +44,7 @@ const EXPECT = {
   '.ico': 'image/x-icon',
   '.pdf': 'application/pdf',
   '.webp': 'image/webp',
-}
+} as Record<string, string>
 
 const CRITICAL = 'critical'
 const WARNING = 'warning'
@@ -56,7 +57,7 @@ program
 
 main(program.opts())
 
-async function main(opts) {
+async function main(opts: { dryRun: boolean; verbose: boolean }) {
   let errors = 0
 
   const files = walk(ASSETS_ROOT, { includeBasePath: true, directories: false }).filter(
@@ -71,7 +72,11 @@ async function main(opts) {
       )
     },
   )
-  const results = (await Promise.all(files.map(checkFile))).filter(Boolean)
+  const results = (await Promise.all(files.map(checkFile))).filter(Boolean) as [
+    level: string,
+    filePath: string,
+    error: string,
+  ][]
   for (const [level, filePath, error] of results) {
     console.log(
       level === CRITICAL ? chalk.red(level) : chalk.yellow(level),
@@ -94,7 +99,7 @@ async function main(opts) {
   process.exitCode = errors
 }
 
-async function checkFile(filePath) {
+async function checkFile(filePath: string) {
   const ext = path.extname(filePath)
 
   const { size } = await fs.stat(filePath)
@@ -113,7 +118,7 @@ async function checkFile(filePath) {
     }
     try {
       checkSVGContent(content)
-    } catch (error) {
+    } catch (error: any) {
       return [CRITICAL, filePath, error.message]
     }
   } else if (EXPECT[ext]) {
@@ -135,15 +140,15 @@ async function checkFile(filePath) {
   // All is well. Nothing to complain about.
 }
 
-function checkSVGContent(content) {
+function checkSVGContent(content: string) {
   const $ = cheerio.load(content)
   const disallowedTagNames = new Set(['script', 'object', 'iframe', 'embed'])
   $('*').each((i, element) => {
-    const { tagName } = element
+    const { tagName } = $(element).get(0)
     if (disallowedTagNames.has(tagName)) {
       throw new Error(`contains a <${tagName}> tag`)
     }
-    for (const key in element.attribs) {
+    for (const key in $(element).get(0).attribs) {
       // Looks for suspicious event handlers on tags.
       // For example `<path oNload="alert(1)"" d="M28 0l4.59 4.59-9.76`
       // We don't need to do a case-sensitive regex here because cheerio

src/assets/tests/static-assets.js → src/assets/tests/static-assets.ts

@@ -6,7 +6,7 @@ import { describe, expect, test, vi } from 'vitest'
 import { get } from '#src/tests/helpers/e2etest.js'
 import { checkCachingHeaders } from '#src/tests/helpers/caching-headers.js'
 
-function getNextStaticAsset(directory) {
+function getNextStaticAsset(directory: string) {
   const root = path.join('.next', 'static', directory)
   const files = fs.readdirSync(root)
   if (!files.length) throw new Error(`Can't find any files in ${root}`)

src/frame/middleware/api.ts

@@ -3,6 +3,7 @@ import { createProxyMiddleware } from 'http-proxy-middleware'
 
 import events from '@/events/middleware.js'
 import anchorRedirect from '@/rest/api/anchor-redirect.js'
+import aiSearch from '@/search/middleware/ai-search'
 import search from '@/search/middleware/search-routes.js'
 import pageInfo from '@/pageinfo/middleware'
 import pageList from '@/pagelist/middleware'
@@ -23,6 +24,23 @@ router.use('/pagelist', pageList)
 // local laptop, they don't have an Elasticsearch. Neither a running local
 // server or the known credentials to a remote Elasticsearch. Whenever
 // that's the case, they can just HTTP proxy to the production server.
+if (process.env.CSE_COPILOT_ENDPOINT || process.env.NODE_ENV === 'test') {
+  router.use('/ai-search', aiSearch)
+} else {
+  console.log(
+    'Proxying AI Search requests to docs.github.com. To use the cse-copilot endpoint, set the CSE_COPILOT_ENDPOINT environment variable.',
+  )
+  router.use(
+    '/ai-search',
+    createProxyMiddleware({
+      target: 'https://docs.github.com',
+      changeOrigin: true,
+      pathRewrite: function (path, req: ExtendedRequest) {
+        return req.originalUrl
+      },
+    }),
+  )
+}
 if (process.env.ELASTICSEARCH_URL) {
   router.use('/search', search)
 } else {

src/landings/components/CategoryLanding.tsx

@@ -4,40 +4,72 @@ import cx from 'classnames'
 import { CookBookArticleCard } from './CookBookArticleCard'
 import { CookBookFilter } from './CookBookFilter'
 
-//import { useTranslation } from 'src/languages/components/useTranslation'
 import { DefaultLayout } from 'src/frame/components/DefaultLayout'
 import { ArticleTitle } from 'src/frame/components/article/ArticleTitle'
 import { Lead } from 'src/frame/components/ui/Lead'
 import { useCategoryLandingContext } from 'src/frame/components/context/CategoryLandingContext'
 import { ClientSideRedirects } from 'src/rest/components/ClientSideRedirects'
 import { RestRedirect } from 'src/rest/components/RestRedirect'
 import { Breadcrumbs } from 'src/frame/components/page-header/Breadcrumbs'
-import { ArticleCardItems } from '../types'
+import { ArticleCardItems } from 'src/landings/types'
 
 export const CategoryLanding = () => {
   const router = useRouter()
-  //const { t } = useTranslation('toc')
   const { title, intro, tocItems } = useCategoryLandingContext()
 
   // tocItems contains directories and its children, we only want the child articles
   const onlyFlatItems: ArticleCardItems = tocItems.flatMap((item) => item.childTocItems || [])
 
-  const [searchResults, setSearchResults] = useState<ArticleCardItems>(onlyFlatItems)
+  const [searchQuery, setSearchQuery] = useState('')
+  const [selectedCategory, setSelectedCategory] = useState('All')
+  const [selectedComplexity, setSelectedComplexity] = useState('All')
 
-  const handleSearch = (query: string) => {
-    const results = onlyFlatItems.filter((token) => {
-      return Object.values(token).some((value) => {
-        if (typeof value === 'string') {
-          return value.toLowerCase().includes(query.toLowerCase())
-        } else if (Array.isArray(value)) {
-          return value.some((item) => item.toLowerCase().includes(query.toLowerCase()))
-        }
-        return false
+  const applyFilters = () => {
+    let results = onlyFlatItems
+
+    if (searchQuery) {
+      results = results.filter((token) => {
+        return Object.values(token).some((value) => {
+          if (typeof value === 'string') {
+            return value.toLowerCase().includes(searchQuery.toLowerCase())
+          } else if (Array.isArray(value)) {
+            return value.some((item) => item.toLowerCase().includes(searchQuery.toLowerCase()))
+          }
+          return false
+        })
       })
-    })
-    setSearchResults(results)
+    }
+
+    if (selectedCategory !== 'All') {
+      results = results.filter((item) => item.category?.includes(selectedCategory))
+    }
+
+    if (selectedComplexity !== 'All') {
+      results = results.filter((item) => item.complexity?.includes(selectedComplexity))
+    }
+
+    return results
+  }
+
+  const searchResults = applyFilters()
+
+  const handleSearch = (query: string) => {
+    setSearchQuery(query)
   }
 
+  const handleFilter = (option: string, type: 'category' | 'complexity') => {
+    if (type === 'category') {
+      setSelectedCategory(option)
+    } else if (type === 'complexity') {
+      setSelectedComplexity(option)
+    }
+  }
+
+  const handleResetFilter = () => {
+    setSearchQuery('')
+    setSelectedCategory('All')
+    setSelectedComplexity('All')
+  }
   return (
     <DefaultLayout>
       {router.route === '/[versionId]/rest/[category]' && <RestRedirect />}
@@ -65,7 +97,12 @@ export const CategoryLanding = () => {
               <h2>Explore {searchResults.length} prompt articles</h2>
             </div>
             <div>
-              <CookBookFilter tokens={onlyFlatItems} onSearch={handleSearch} />
+              <CookBookFilter
+                tokens={onlyFlatItems}
+                onSearch={handleSearch}
+                handleFilter={handleFilter}
+                handleResetFilter={handleResetFilter}
+              />
             </div>
           </div>
           <ul className="clearfix gutter-md-spacious">
@@ -80,6 +117,7 @@ export const CategoryLanding = () => {
                     ...(item.category || []),
                     ...(item.complexity || []),
                   ]}
+                  url={item.fullPath}
                 />
               </li>
             ))}

src/landings/components/CookBookArticleCard.tsx

@@ -1,4 +1,4 @@
-import { Label, LabelGroup } from '@primer/react'
+import { Label, LabelGroup, Link } from '@primer/react'
 import { BugIcon } from '@primer/octicons-react'
 
 type Props = {
@@ -55,6 +55,7 @@ export const CookBookArticleCard = ({
   tags = defaultProps.tags,
   description = defaultProps.description,
   image = '',
+  url,
   spotlight = false,
 }: Props) => {
   return (
@@ -66,7 +67,9 @@ export const CookBookArticleCard = ({
         {spotlight ? setImage(image) : null}
         {spotlight ? setIcon('none') : setIcon(icon)}
         <div>
-          <h3 className="h4">{title}</h3>
+          <Link href={url}>
+            <h3 className="h4">{title}</h3>
+          </Link>
           <div className="fgColor-muted mb-3 mt-2">{description}</div>
           <LabelGroup>
             {tags.map((tag, index) => (

src/landings/components/CookBookFilter.tsx

@@ -1,16 +1,52 @@
-import { TextInput, ActionMenu, ActionList, Button } from '@primer/react'
+import { TextInput, ActionMenu, ActionList, Button, Box } from '@primer/react'
 import { SearchIcon } from '@primer/octicons-react'
-import { useRef, useEffect } from 'react'
+import { useRef, useEffect, useState } from 'react'
 import { ArticleCardItems } from '#src/landings/types.ts'
 
 type Props = {
   tokens: ArticleCardItems
   onSearch: (query: string) => void
   isSearchOpen?: boolean
+  handleFilter: (option: string, type: 'category' | 'complexity') => void
+  handleResetFilter: () => void
 }
 
-export const CookBookFilter = ({ onSearch, isSearchOpen }: Props) => {
+export const CookBookFilter = ({
+  onSearch,
+  isSearchOpen,
+  tokens,
+  handleFilter,
+  handleResetFilter,
+}: Props) => {
+  const categories: string[] = ['All', ...new Set(tokens.flatMap((item) => item.category || []))]
+  const complexities: string[] = [
+    'All',
+    ...new Set(tokens.flatMap((item) => item.complexity || [])),
+  ]
+
+  const [selectedCategory, setSelectedCategory] = useState(0)
+  const [selectedComplexity, setSelectedComplexity] = useState(0)
+
   const inputRef = useRef<HTMLInputElement>(null)
+
+  const onFilter = (option: string, type: 'category' | 'complexity', index: number) => {
+    if (type === 'category') {
+      setSelectedCategory(index)
+    } else if (type === 'complexity') {
+      setSelectedComplexity(index)
+    }
+    handleFilter(option, type)
+  }
+
+  const onResetFilter = () => {
+    setSelectedCategory(0)
+    setSelectedComplexity(0)
+    handleResetFilter()
+    if (inputRef.current) {
+      inputRef.current.value = ''
+    }
+  }
+
   useEffect(() => {
     if (isSearchOpen) {
       inputRef.current?.focus()
@@ -38,43 +74,63 @@ export const CookBookFilter = ({ onSearch, isSearchOpen }: Props) => {
       <div className="d-flex">
         <ActionMenu>
           <ActionMenu.Button className="col-md-1 col-sm-2 float-md-left m-1">
-            Category
+            <Box
+              sx={{
+                color: 'fg.muted',
+                display: 'inline-block',
+              }}
+            >
+              Category:
+            </Box>{' '}
+            {categories[selectedCategory]}
           </ActionMenu.Button>
-          <ActionMenu.Overlay width="small">
-            <ActionList>
-              <ActionList.Item>Item 1</ActionList.Item>
-              <ActionList.Item>Item 2</ActionList.Item>
-              <ActionList.Item>Item 3</ActionList.Item>
+          <ActionMenu.Overlay width="auto">
+            <ActionList selectionVariant="single">
+              {categories.map((category, index) => (
+                <ActionList.Item
+                  key={index}
+                  selected={index === selectedCategory}
+                  onSelect={() => onFilter(category, 'category', index)}
+                >
+                  {category}
+                </ActionList.Item>
+              ))}
             </ActionList>
           </ActionMenu.Overlay>
         </ActionMenu>
 
         <ActionMenu>
-          <ActionMenu.Button className="col-md-1 col-sm-2 float-left m-1">
-            Complexity
+          <ActionMenu.Button className="col-md-1 col-sm-2 float-md-left m-1">
+            <Box
+              sx={{
+                color: 'fg.muted',
+                display: 'inline-block',
+              }}
+            >
+              Complexity:
+            </Box>{' '}
+            {complexities[selectedComplexity]}
           </ActionMenu.Button>
-          <ActionMenu.Overlay width="small">
-            <ActionList>
-              <ActionList.Item>Item 1</ActionList.Item>
-              <ActionList.Item>Item 2</ActionList.Item>
-              <ActionList.Item>Item 3</ActionList.Item>
+          <ActionMenu.Overlay width="auto">
+            <ActionList selectionVariant="single">
+              {complexities.map((complexity, index) => (
+                <ActionList.Item
+                  key={index}
+                  selected={index === selectedComplexity}
+                  onSelect={() => onFilter(complexity, 'complexity', index)}
+                >
+                  {complexity}
+                </ActionList.Item>
+              ))}
             </ActionList>
           </ActionMenu.Overlay>
         </ActionMenu>
 
-        <ActionMenu>
-          <ActionMenu.Button className="col-md-1 col-sm-2 float-left m-1">
-            Industry
-          </ActionMenu.Button>
-          <ActionMenu.Overlay width="small">
-            <ActionList>
-              <ActionList.Item>Item 1</ActionList.Item>
-              <ActionList.Item>Item 2</ActionList.Item>
-              <ActionList.Item>Item 3</ActionList.Item>
-            </ActionList>
-          </ActionMenu.Overlay>
-        </ActionMenu>
-        <Button variant="invisible" className="col-md-1 col-sm-2 float-left mt-1">
+        <Button
+          variant="invisible"
+          className="col-md-1 col-sm-2 float-left mt-1"
+          onClick={onResetFilter}
+        >
           Reset filters
         </Button>
       </div>

src/search/lib/ai-search-proxy.ts

@@ -0,0 +1,78 @@
+import { Request, Response } from 'express'
+import got from 'got'
+import { getHmacWithEpoch } from '@/search/lib/helpers/get-cse-copilot-auth'
+import { getCSECopilotSource } from '#src/search/lib/helpers/cse-copilot-docs-versions.js'
+
+export const aiSearchProxy = async (req: Request, res: Response) => {
+  const { query, version, language } = req.body
+  const errors = []
+
+  // Validate request body
+  if (!query) {
+    errors.push({ message: `Missing required key 'query' in request body` })
+  } else if (typeof query !== 'string') {
+    errors.push({ message: `Invalid 'query' in request body. Must be a string` })
+  }
+  if (!version) {
+    errors.push({ message: `Missing required key 'version' in request body` })
+  }
+  if (!language) {
+    errors.push({ message: `Missing required key 'language' in request body` })
+  }
+
+  let docsSource = ''
+  try {
+    docsSource = getCSECopilotSource(version, language)
+  } catch (error: any) {
+    errors.push({ message: error?.message || 'Invalid version or language' })
+  }
+
+  if (errors.length) {
+    res.status(400).json({ errors })
+    return
+  }
+
+  const body = {
+    chat_context: 'defaults',
+    docs_source: docsSource,
+    query,
+    stream: true,
+  }
+
+  try {
+    const stream = got.post(`${process.env.CSE_COPILOT_ENDPOINT}/answers`, {
+      json: body,
+      headers: {
+        Authorization: getHmacWithEpoch(),
+        'Content-Type': 'application/json',
+      },
+      isStream: true,
+    })
+
+    // Set response headers
+    res.setHeader('Content-Type', 'application/x-ndjson')
+    res.flushHeaders()
+
+    // Pipe the got stream directly to the response
+    stream.pipe(res)
+
+    // Handle stream errors
+    stream.on('error', (error) => {
+      console.error('Error streaming from cse-copilot:', error)
+      // Only send error response if headers haven't been sent
+      if (!res.headersSent) {
+        res.status(500).json({ errors: [{ message: 'Internal server error' }] })
+      } else {
+        res.end()
+      }
+    })
+
+    // Ensure response ends when stream ends
+    stream.on('end', () => {
+      res.end()
+    })
+  } catch (error) {
+    console.error('Error posting /answers to cse-copilot:', error)
+    res.status(500).json({ errors: [{ message: 'Internal server error' }] })
+  }
+}

src/search/lib/helpers/cse-copilot-docs-versions.ts

@@ -0,0 +1,44 @@
+// Versions used by cse-copilot
+import { allVersions } from '@/versions/lib/all-versions'
+const CSE_COPILOT_DOCS_VERSIONS = ['dotcom', 'ghec', 'ghes']
+
+// Languages supported by cse-copilot
+const DOCS_LANGUAGES = ['en']
+export function supportedCSECopilotLanguages() {
+  return DOCS_LANGUAGES
+}
+
+export function getCSECopilotSource(
+  version: (typeof CSE_COPILOT_DOCS_VERSIONS)[number],
+  language: (typeof DOCS_LANGUAGES)[number],
+) {
+  const cseCopilotDocsVersion = getMiscBaseNameFromVersion(version)
+  if (!CSE_COPILOT_DOCS_VERSIONS.includes(cseCopilotDocsVersion)) {
+    throw new Error(
+      `Invalid 'version' in request body: '${version}'. Must be one of: ${CSE_COPILOT_DOCS_VERSIONS.join(', ')}`,
+    )
+  }
+  if (!DOCS_LANGUAGES.includes(language)) {
+    throw new Error(
+      `Invalid 'language' in request body '${language}'. Must be one of: ${DOCS_LANGUAGES.join(', ')}`,
+    )
+  }
+  return `docs_${version}_${language}`
+}
+
+function getMiscBaseNameFromVersion(Version: string): string {
+  const miscBaseName =
+    Object.values(allVersions).find(
+      (info) =>
+        info.shortName === Version ||
+        info.plan === Version ||
+        info.miscVersionName === Version ||
+        info.currentRelease === Version,
+    )?.miscBaseName || ''
+
+  if (!miscBaseName) {
+    return ''
+  }
+
+  return miscBaseName
+}

src/search/lib/helpers/get-cse-copilot-auth.ts

@@ -0,0 +1,24 @@
+import crypto from 'crypto'
+
+// github/cse-copilot's API requires an HMAC-SHA256 signature with each request
+export function getHmacWithEpoch() {
+  const epochTime = getEpochTime().toString()
+  // CSE_COPILOT_SECRET needs to be set for the api-ai-search tests to work
+  if (process.env.NODE_ENV === 'test') {
+    process.env.CSE_COPILOT_SECRET = 'mock-secret'
+  }
+  if (!process.env.CSE_COPILOT_SECRET) {
+    throw new Error('CSE_COPILOT_SECRET is not defined')
+  }
+  const hmac = generateHmacSha256(process.env.CSE_COPILOT_SECRET, epochTime)
+  return `${epochTime}.${hmac}`
+}
+
+// In seconds
+function getEpochTime(): number {
+  return Math.floor(Date.now() / 1000)
+}
+
+function generateHmacSha256(key: string, data: string): string {
+  return crypto.createHmac('sha256', key).update(data).digest('hex')
+}

src/search/middleware/ai-search.ts

@@ -0,0 +1,20 @@
+import express, { Request, Response } from 'express'
+
+import catchMiddlewareError from '#src/observability/middleware/catch-middleware-error.js'
+import { aiSearchProxy } from '../lib/ai-search-proxy'
+
+const router = express.Router()
+
+router.post(
+  '/v1',
+  catchMiddlewareError(async (req: Request, res: Response) => {
+    await aiSearchProxy(req, res)
+  }),
+)
+
+// Redirect to most recent version
+router.post('/', (req, res) => {
+  res.redirect(307, req.originalUrl.replace('/ai-search', '/ai-search/v1'))
+})
+
+export default router

src/search/tests/api-ai-search.ts

@@ -0,0 +1,148 @@
+import { expect, test, describe, beforeAll, afterAll } from 'vitest'
+
+import { post } from 'src/tests/helpers/e2etest.js'
+import { startMockServer, stopMockServer } from '@/tests/mocks/start-mock-server'
+
+describe('AI Search Routes', () => {
+  beforeAll(() => {
+    startMockServer()
+  })
+  afterAll(() => stopMockServer())
+
+  test('/api/ai-search/v1 should handle a successful response', async () => {
+    let apiBody = { query: 'How do I create a Repository?', language: 'en', version: 'dotcom' }
+
+    const response = await fetch('http://localhost:4000/api/ai-search/v1', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(apiBody),
+    })
+
+    expect(response.ok).toBe(true)
+    expect(response.headers.get('content-type')).toBe('application/x-ndjson')
+    expect(response.headers.get('transfer-encoding')).toBe('chunked')
+
+    if (!response.body) {
+      throw new Error('ReadableStream not supported in this environment.')
+    }
+
+    const decoder = new TextDecoder('utf-8')
+    const reader = response.body.getReader()
+    let done = false
+    const chunks = []
+
+    while (!done) {
+      const { value, done: readerDone } = await reader.read()
+      done = readerDone
+
+      if (value) {
+        // Decode the Uint8Array chunk into a string
+        const chunkStr = decoder.decode(value, { stream: true })
+        chunks.push(chunkStr)
+      }
+    }
+
+    // Combine all chunks into a single string
+    const fullResponse = chunks.join('')
+    // Split the response into individual chunk lines
+    const chunkLines = fullResponse.split('\n').filter((line) => line.trim() !== '')
+
+    // Assertions:
+
+    // 1. First chunk should be the SOURCES chunk
+    expect(chunkLines.length).toBeGreaterThan(0)
+    const firstChunkMatch = chunkLines[0].match(/^Chunk: (.+)$/)
+    expect(firstChunkMatch).not.toBeNull()
+
+    const sourcesChunk = JSON.parse(firstChunkMatch?.[1] || '')
+    expect(sourcesChunk).toHaveProperty('chunkType', 'SOURCES')
+    expect(sourcesChunk).toHaveProperty('sources')
+    expect(Array.isArray(sourcesChunk.sources)).toBe(true)
+    expect(sourcesChunk.sources.length).toBe(3)
+
+    // 2. Subsequent chunks should be MESSAGE_CHUNKs
+    for (let i = 1; i < chunkLines.length; i++) {
+      const line = chunkLines[i]
+      const messageChunk = JSON.parse(line)
+      expect(messageChunk).toHaveProperty('chunkType', 'MESSAGE_CHUNK')
+      expect(messageChunk).toHaveProperty('text')
+      expect(typeof messageChunk.text).toBe('string')
+    }
+
+    // 3. Verify the complete message is expected
+    const expectedMessage =
+      'Creating a repository on GitHub is something you should already know how to do :shrug:'
+    const receivedMessage = chunkLines
+      .slice(1)
+      .map((line) => JSON.parse(line).text)
+      .join('')
+    expect(receivedMessage).toBe(expectedMessage)
+  })
+
+  test('should handle validation errors: query missing', async () => {
+    let body = { language: 'en', version: 'dotcom' }
+    const response = await post('/api/ai-search/v1', {
+      body: JSON.stringify(body),
+      headers: { 'Content-Type': 'application/json' },
+    })
+
+    const responseBody = JSON.parse(response.body)
+
+    expect(response.ok).toBe(false)
+    expect(responseBody['errors']).toEqual([
+      { message: `Missing required key 'query' in request body` },
+    ])
+  })
+
+  test('should handle validation errors: language missing', async () => {
+    let body = { query: 'example query', version: 'dotcom' }
+    const response = await post('/api/ai-search/v1', {
+      body: JSON.stringify(body),
+      headers: { 'Content-Type': 'application/json' },
+    })
+
+    const responseBody = JSON.parse(response.body)
+
+    expect(response.ok).toBe(false)
+    expect(responseBody['errors']).toEqual([
+      { message: `Missing required key 'language' in request body` },
+      { message: `Invalid 'language' in request body 'undefined'. Must be one of: en` },
+    ])
+  })
+
+  test('should handle validation errors: version missing', async () => {
+    let body = { query: 'example query', language: 'en' }
+    const response = await post('/api/ai-search/v1', {
+      body: JSON.stringify(body),
+      headers: { 'Content-Type': 'application/json' },
+    })
+
+    const responseBody = JSON.parse(response.body)
+
+    expect(response.ok).toBe(false)
+    expect(responseBody['errors']).toEqual([
+      { message: `Missing required key 'version' in request body` },
+      {
+        message: `Invalid 'version' in request body: 'undefined'. Must be one of: dotcom, ghec, ghes`,
+      },
+    ])
+  })
+
+  test('should handle multiple validation errors: query missing, invalid language and version', async () => {
+    let body = { language: 'fr', version: 'fpt' }
+    const response = await post('/api/ai-search/v1', {
+      body: JSON.stringify(body),
+      headers: { 'Content-Type': 'application/json' },
+    })
+
+    const responseBody = JSON.parse(response.body)
+
+    expect(response.ok).toBe(false)
+    expect(responseBody['errors']).toEqual([
+      { message: `Missing required key 'query' in request body` },
+      {
+        message: `Invalid 'language' in request body 'fr'. Must be one of: en`,
+      },
+    ])
+  })
+})

src/tests/mocks/cse-copilot-mock.ts

@@ -0,0 +1,71 @@
+import { Request, Response } from 'express'
+
+// Prefix used for mocking. This can be any value
+export const CSE_COPILOT_PREFIX = 'cse-copilot'
+
+export function cseCopilotPostAnswersMock(req: Request, res: Response) {
+  // Set headers for chunked transfer and encoding
+  res.setHeader('Content-Type', 'application/json; charset=utf-8')
+  res.setHeader('Transfer-Encoding', 'chunked')
+
+  // Define the SOURCES chunk
+  const sourcesChunk = {
+    chunkType: 'SOURCES',
+    sources: [
+      {
+        title: 'Creating a new repository',
+        url: 'https://docs.github.com/en/repositories/creating-and-managing-repositories/creating-a-new-repository',
+        index: '/en/repositories/creating-and-managing-repositories/creating-a-new-repository',
+      },
+      {
+        title: 'Creating and managing repositories',
+        url: 'https://docs.github.com/en/repositories/creating-and-managing-repositories',
+        index: '/en/repositories/creating-and-managing-repositories',
+      },
+      {
+        title: 'GitHub Terms of Service',
+        url: 'https://docs.github.com/en/site-policy/github-terms/github-terms-of-service',
+        index: '/en/site-policy/github-terms/github-terms-of-service',
+      },
+    ],
+  }
+
+  // Function to send a chunk with proper encoding
+  const sendEncodedChunk = (data: any, isLast = false) => {
+    const prefix = isLast ? '' : '\n' // Optionally, add delimiters if needed
+    const buffer = Buffer.from(prefix + data, 'utf-8')
+    res.write(buffer)
+  }
+
+  // Send the SOURCES chunk
+  sendEncodedChunk(`Chunk: ${JSON.stringify(sourcesChunk)}\n\n`)
+
+  // Define the message to be sent in chunks
+  const message =
+    'Creating a repository on GitHub is something you should already know how to do :shrug:'
+
+  // Split the message into words (or adjust the splitting logic as needed)
+  const words = message.split(' ')
+
+  let index = 0
+
+  const sendChunk = () => {
+    if (index < words.length) {
+      const word = words[index]
+      const isLastWord = index === words.length - 1
+      const chunk = {
+        chunkType: 'MESSAGE_CHUNK',
+        text: word + (isLastWord ? '' : ' '), // Add space if not the last word
+      }
+      sendEncodedChunk(`${JSON.stringify(chunk)}\n`)
+      index++
+      sendChunk() // Adjust the delay as needed
+    } else {
+      // End the response after all chunks are sent
+      res.end()
+    }
+  }
+
+  // Start sending MESSAGE_CHUNKs
+  sendChunk()
+}

src/tests/mocks/start-mock-server.ts

@@ -0,0 +1,73 @@
+/* When testing API routes via an integration test, e.g.
+
+const res = await post('/api/<some-route>', {
+  body: JSON.stringify(api_body),
+  headers: { 'Content-Type': 'application/json' },
+})
+
+expect(res.status).toBe(200)
+
+The `api/<route>` may call an external URL.
+
+We are unable to use `nock` in this circumstance since we run the server in a separate instance.
+
+Instead, we can use the `startMockServer` helper to start a mock server that will intercept the request and return a canned response.
+
+In order for this to work you MUST use a process.env variable for the URL you are calling, 
+
+e.g. `process.env.CSE_COPILOT_ENDPOINT`
+
+You should override the variable  in the overrideEnvForTesting function in this file.
+*/
+
+import express from 'express'
+import { CSE_COPILOT_PREFIX, cseCopilotPostAnswersMock } from './cse-copilot-mock'
+
+// Define the default port for the mock server
+const MOCK_SERVER_PORT = 3012
+
+// Construct the server URL using the defined port
+const serverUrl = `http://localhost:${MOCK_SERVER_PORT}`
+
+// Variable to hold the server instance
+let server: any = null
+
+// Override environment variables for testing purposes
+export function overrideEnvForTesting() {
+  process.env.CSE_COPILOT_ENDPOINT = `${serverUrl}/${CSE_COPILOT_PREFIX}`
+}
+
+// Function to start the mock server
+export function startMockServer(port = MOCK_SERVER_PORT) {
+  const app = express()
+  app.use(express.json())
+
+  // Define your mock routes here
+  app.post(`/${CSE_COPILOT_PREFIX}/answers`, cseCopilotPostAnswersMock)
+
+  // Start the server and store the server instance
+  server = app.listen(port, () => {
+    console.log(`Mock server is running on port ${port}`)
+  })
+}
+
+// Function to stop the mock server
+export function stopMockServer(): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (server) {
+      server.close((err: any) => {
+        if (err) {
+          console.error('Error stopping the mock server:', err)
+          reject(err)
+        } else {
+          console.log('Mock server has been stopped.')
+          server = null
+          resolve()
+        }
+      })
+    } else {
+      console.warn('Mock server is not running.')
+      resolve()
+    }
+  })
+}

src/tests/vitest.setup.ts

@@ -1,4 +1,5 @@
 import { main } from 'src/frame/start-server'
+import { overrideEnvForTesting } from './mocks/start-mock-server'
 
 let teardownHappened = false
 type PromiseType<T extends Promise<any>> = T extends Promise<infer U> ? U : never
@@ -7,6 +8,7 @@ type Server = PromiseType<ReturnType<typeof main>>
 let server: Server | undefined
 
 export async function setup() {
+  overrideEnvForTesting()
   server = await main()
 }
 

tsconfig.json

@@ -1,11 +1,7 @@
 {
   "compilerOptions": {
     "target": "ES2022",
-    "lib": [
-      "dom",
-      "dom.iterable",
-      "esnext"
-    ],
+    "lib": ["dom", "dom.iterable", "esnext"],
     "allowJs": true,
     "skipLibCheck": true,
     "strict": true,
@@ -30,9 +26,5 @@
     "docs-internal-data",
     "src/code-scanning/scripts/generate-code-scanning-query-list.ts"
   ],
-  "include": [
-    "*.d.ts",
-    "**/*.ts",
-    "**/*.tsx"
-  ]
+  "include": ["*.d.ts", "**/*.ts", "**/*.tsx"]
 }