✨ Site Nav from tag graph

adminSiteServer/mockSiteRouter.ts

@@ -570,6 +570,15 @@ getPlainRouteWithROTransaction(
     }
 )
 
+getPlainRouteWithROTransaction(
+    mockSiteRouter,
+    "/topicTagGraph.json",
+    async (req, res, trx) => {
+        const headerMenu = await db.generateTopicTagGraph(trx)
+        res.send(headerMenu)
+    }
+)
+
 getPlainRouteWithROTransaction(mockSiteRouter, "/*", async (req, res, trx) => {
     // Remove leading and trailing slashes
     const slug = req.path.replace(/^\/|\/$/g, "")

baker/SiteBaker.tsx

@@ -1087,6 +1087,13 @@ export class SiteBaker {
             `rm -rf ${this.bakedSiteDir}/assets && cp -r ${BASE_DIR}/dist/assets ${this.bakedSiteDir}/assets`
         )
 
+        await fs.writeFile(
+            `${this.bakedSiteDir}/topicTagGraph.json`,
+            await db
+                .generateTopicTagGraph(trx)
+                .then((nav) => JSON.stringify(nav))
+        )
+
         // The `assets-admin` folder is optional; don't fail if it doesn't exist
         await execWrapper(
             `rm -rf ${this.bakedSiteDir}/assets-admin && (cp -r ${BASE_DIR}/dist/assets-admin ${this.bakedSiteDir}/assets-admin || true)`

db/db.ts

@@ -33,6 +33,7 @@ import {
     DbEnrichedImageWithUserId,
     MinimalTag,
     BreadcrumbItem,
+    TagGraphRoot,
 } from "@ourworldindata/types"
 import { groupBy } from "lodash"
 import { gdocFromJSON } from "./model/Gdoc/GdocFactory.js"
@@ -507,6 +508,7 @@ export async function getFlatTagGraph(knex: KnexReadonlyTransaction): Promise<
             tg.parentId,
             tg.childId,
             tg.weight,
+            t.slug,
             t.name,
             p.slug IS NOT NULL AS isTopic
         FROM
@@ -858,3 +860,43 @@ export function getImageUsage(trx: KnexReadonlyTransaction): Promise<
         )
     )
 }
+
+export async function generateTopicTagGraph(
+    knex: KnexReadonlyTransaction
+): Promise<TagGraphRoot> {
+    const { __rootId, ...parents } = await getFlatTagGraph(knex)
+
+    const tagGraphTopicsOnly = Object.entries(parents).reduce(
+        (acc: FlatTagGraph, [parentId, children]) => {
+            acc[Number(parentId)] = children.filter((child) => {
+                if (child.parentId === __rootId) return true
+                return child.isTopic
+            })
+            return acc
+        },
+        {} as FlatTagGraph
+    )
+
+    return createTagGraph(tagGraphTopicsOnly, __rootId)
+}
+
+export const getUniqueTopicCount = (
+    trx: KnexReadonlyTransaction
+): Promise<number> => {
+    const count = knexRawFirst<{ count: number }>(
+        trx,
+        `-- sql
+        SELECT COUNT(DISTINCT(t.slug))
+        FROM tags t
+        LEFT JOIN posts_gdocs p ON t.slug = p.slug
+        WHERE t.slug IS NOT NULL AND p.published IS TRUE`
+    )
+        .then((res) => (res ? res.count : 0))
+        .catch((e) => {
+            console.error("Failed to get unique topic count", e)
+            throw e
+        })
+    // throw on count == 0 also
+    if (!count) throw new Error("Failed to get unique topic count")
+    return count
+}

db/model/Gdoc/GdocHomepage.ts

@@ -12,7 +12,6 @@ import {
     OwidGdocBaseInterface,
     OwidGdocHomepageMetadata,
 } from "@ourworldindata/types"
-import { getUniqueTopicCount } from "../../../site/gdocs/utils.js"
 import { getLatestDataInsights } from "./GdocFactory.js"
 
 export class GdocHomepage
@@ -69,7 +68,8 @@ export class GdocHomepage
 
         this.homepageMetadata = {
             chartCount: grapherCount + nonGrapherExplorerViewCount,
-            topicCount: getUniqueTopicCount(),
+            topicCount: await db.getUniqueTopicCount(knex),
+            tagGraph: await db.generateTopicTagGraph(knex),
         }
 
         const { dataInsights, imageMetadata } =

db/model/Post.ts

@@ -6,7 +6,6 @@ import {
     parsePostWpApiSnapshot,
     FullPost,
     JsonError,
-    CategoryWithEntries,
     WP_PostType,
     FilterFnPostRestApi,
     PostRestApi,
@@ -40,7 +39,6 @@ import {
     CLOUDFLARE_IMAGES_URL,
 } from "../../settings/clientSettings.js"
 import { BLOG_SLUG } from "../../settings/serverSettings.js"
-import { SiteNavigationStatic } from "../../site/SiteConstants.js"
 import { decodeHTML } from "entities"
 import { getAndLoadListedGdocPosts } from "./Gdoc/GdocFactory.js"
 
@@ -185,22 +183,9 @@ export const getFullPostByIdFromSnapshot = async (
     return getFullPost(trx, postEnriched.wpApiSnapshot)
 }
 
-// TODO: I suggest that in the place where we define SiteNavigationStatic we create a Set with all the leaves and
-//       then this one becomes a simple lookup in the set. Probably nicest to do the set creation as a memoized function.
+// This function used to be more complicated, but now the only citable WP post is the COVID page
 export const isPostSlugCitable = (slug: string): boolean => {
-    const entries = SiteNavigationStatic.categories
-    return entries.some((category) => {
-        return (
-            category.entries.some((entry) => entry.slug === slug) ||
-            (category.subcategories ?? []).some(
-                (subcategory: CategoryWithEntries) => {
-                    return subcategory.entries.some(
-                        (subCategoryEntry) => subCategoryEntry.slug === slug
-                    )
-                }
-            )
-        )
-    })
+    return slug === "coronavirus"
 }
 
 export const getPostsFromSnapshots = async (

packages/@ourworldindata/types/src/domainTypes/ContentGraph.ts

@@ -30,6 +30,7 @@ export type FlatTagGraphNode = Pick<DbPlainTag, "name" | "slug"> & {
     isTopic: boolean
     parentId: number
     childId: number
+    slug: string | null
 }
 
 export type FlatTagGraph = Record<number, FlatTagGraphNode[]>

packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts

@@ -14,6 +14,7 @@ import {
 import { MinimalTag } from "../dbTypes/Tags.js"
 import { DbEnrichedLatestWork } from "../domainTypes/Author.js"
 import { QueryParams } from "../domainTypes/Various.js"
+import { TagGraphRoot } from "../domainTypes/ContentGraph.js"
 
 export enum OwidGdocPublicationContext {
     unlisted = "unlisted",
@@ -189,6 +190,7 @@ export interface OwidGdocHomepageContent {
 export interface OwidGdocHomepageMetadata {
     chartCount?: number
     topicCount?: number
+    tagGraph?: TagGraphRoot
 }
 
 export interface OwidGdocHomepageInterface extends OwidGdocBaseInterface {

packages/@ourworldindata/utils/src/Util.ts

@@ -1981,6 +1981,13 @@ export function createTagGraph(
     return recursivelySetChildren(tagGraph) as TagGraphRoot
 }
 
+export const getAllTopicsInArea = (area: TagGraphNode): TagGraphNode[] => {
+    return [
+        ...area.children,
+        ...area.children.flatMap((child) => getAllTopicsInArea(child)),
+    ]
+}
+
 export function formatInlineList(
     array: unknown[],
     connector: "and" | "or" = "and"

packages/@ourworldindata/utils/src/index.ts

@@ -124,6 +124,7 @@ export {
     commafyNumber,
     isFiniteWithGuard,
     createTagGraph,
+    getAllTopicsInArea,
     formatInlineList,
     lazy,
     getParentVariableIdFromChartConfig,

site/SiteMobileCategory.tsx → site/SiteMobileArea.tsx

@@ -1,17 +1,16 @@
 import { useEffect, useRef } from "react"
-import { CategoryWithEntries } from "@ourworldindata/utils"
-import { SiteNavigationToggle } from "./SiteNavigationToggle.js"
 import { SiteNavigationTopic } from "./SiteNavigationTopic.js"
-import { allTopicsInCategory } from "./gdocs/utils.js"
+import { TagGraphNode, getAllTopicsInArea } from "@ourworldindata/utils"
+import { SiteNavigationToggle } from "./SiteNavigationToggle.js"
 
-export const SiteMobileCategory = ({
-    category,
+export const SiteMobileArea = ({
+    area,
     isActive,
-    toggleCategory,
+    toggleArea,
 }: {
-    category: CategoryWithEntries
+    area: TagGraphNode
     isActive: boolean
-    toggleCategory: (category: CategoryWithEntries) => void
+    toggleArea: (category: TagGraphNode) => void
 }) => {
     const categoryRef = useRef<HTMLLIElement>(null)
 
@@ -22,20 +21,14 @@ export const SiteMobileCategory = ({
     }, [isActive])
 
     return (
-        <li
-            key={category.slug}
-            className="SiteMobileCategory"
-            ref={categoryRef}
-        >
+        <li key={area.slug} className="SiteMobileCategory" ref={categoryRef}>
             <SiteNavigationToggle
-                ariaLabel={
-                    isActive ? `Collapse ${category}` : `Expand ${category}`
-                }
+                ariaLabel={isActive ? `Collapse ${area}` : `Expand ${area}`}
                 isActive={isActive}
-                onToggle={() => toggleCategory(category)}
+                onToggle={() => toggleArea(area)}
                 dropdown={
                     <ul>
-                        {allTopicsInCategory(category).map((topic) => (
+                        {getAllTopicsInArea(area).map((topic) => (
                             <SiteNavigationTopic
                                 key={topic.slug}
                                 topic={topic}
@@ -45,7 +38,7 @@ export const SiteMobileCategory = ({
                 }
                 withCaret={true}
             >
-                {category.name}
+                {area.name}
             </SiteNavigationToggle>
         </li>
     )

site/SiteMobileMenu.tsx

@@ -1,31 +1,30 @@
 import { useState } from "react"
-import { CategoryWithEntries } from "@ourworldindata/utils"
+import { TagGraphNode, TagGraphRoot } from "@ourworldindata/utils"
 import classnames from "classnames"
 import { SiteNavigationToggle } from "./SiteNavigationToggle.js"
 import { Menu } from "./SiteConstants.js"
 import { SiteAbout } from "./SiteAbout.js"
 import { SiteResources } from "./SiteResources.js"
-import { SiteMobileCategory } from "./SiteMobileCategory.js"
+import { SiteMobileArea } from "./SiteMobileArea.js"
 
 export const SiteMobileMenu = ({
-    topics,
+    tagGraph,
     menu,
     toggleMenu,
     className,
 }: {
-    topics: CategoryWithEntries[]
+    tagGraph: TagGraphRoot | null
     menu: Menu | null
     toggleMenu: (menu: Menu) => void
     className?: string
 }) => {
-    const [activeCategory, setActiveCategory] =
-        useState<CategoryWithEntries | null>(null)
+    const [activeArea, setActiveArea] = useState<TagGraphNode | null>(null)
 
-    const toggleCategory = (category: CategoryWithEntries) => {
-        if (activeCategory === category) {
-            setActiveCategory(null)
+    const toggleArea = (area: TagGraphNode) => {
+        if (activeArea === area) {
+            setActiveArea(null)
         } else {
-            setActiveCategory(category)
+            setActiveArea(area)
         }
     }
 
@@ -35,12 +34,12 @@ export const SiteMobileMenu = ({
                 <li>
                     <span className="section__header">Browse by topic</span>
                     <ul className="section__dropdown--topics">
-                        {topics.map((category) => (
-                            <SiteMobileCategory
-                                key={category.slug}
-                                category={category}
-                                isActive={activeCategory === category}
-                                toggleCategory={toggleCategory}
+                        {tagGraph?.children.map((area) => (
+                            <SiteMobileArea
+                                key={area.slug}
+                                area={area}
+                                isActive={activeArea === area}
+                                toggleArea={toggleArea}
                             />
                         ))}
                     </ul>

site/SiteNavigation.tsx

@@ -11,14 +11,15 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome/index.js"
 import { SiteNavigationTopics } from "./SiteNavigationTopics.js"
 import { SiteLogos } from "./SiteLogos.js"
 import { SiteAbout } from "./SiteAbout.js"
+import { TagGraphRoot } from "@ourworldindata/utils"
 import { SiteResources } from "./SiteResources.js"
 import { SiteSearchNavigation } from "./SiteSearchNavigation.js"
 import { SiteMobileMenu } from "./SiteMobileMenu.js"
 import { SiteNavigationToggle } from "./SiteNavigationToggle.js"
 import classnames from "classnames"
 import { useTriggerOnEscape } from "./hooks.js"
 import { AUTOCOMPLETE_CONTAINER_ID } from "./search/Autocomplete.js"
-import { Menu, SiteNavigationStatic } from "./SiteConstants.js"
+import { Menu } from "./SiteConstants.js"
 
 // Note: tranforming the flag from an env string to a boolean in
 // clientSettings.ts is convoluted due to the two-pass SSR/Vite build process.
@@ -35,6 +36,16 @@ export const SiteNavigation = ({
 }) => {
     const [menu, setActiveMenu] = useState<Menu | null>(null)
     const [query, setQuery] = useState<string>("")
+    const [tagGraph, setTagGraph] = useState<TagGraphRoot | null>(null)
+
+    useEffect(() => {
+        const fetchTagGraph = async () => {
+            const response = await fetch("/topicTagGraph.json")
+            const tagGraph = await response.json()
+            setTagGraph(tagGraph)
+        }
+        if (!tagGraph) fetchTagGraph().catch(console.error)
+    }, [tagGraph, setTagGraph])
 
     const isActiveMobileMenu =
         menu !== null &&
@@ -111,7 +122,7 @@ export const SiteNavigation = ({
                                 <SiteMobileMenu
                                     menu={menu}
                                     toggleMenu={toggleMenu}
-                                    topics={SiteNavigationStatic.categories}
+                                    tagGraph={tagGraph}
                                     className="hide-sm-up"
                                 />
                             }
@@ -131,9 +142,7 @@ export const SiteNavigation = ({
                                         dropdown={
                                             <SiteNavigationTopics
                                                 onClose={closeOverlay}
-                                                topics={
-                                                    SiteNavigationStatic.categories
-                                                }
+                                                tagGraph={tagGraph}
                                                 className="hide-sm-only"
                                             />
                                         }

site/SiteNavigationTopic.tsx

@@ -1,10 +1,10 @@
-import { EntryMeta } from "@ourworldindata/utils"
+import { TagGraphNode } from "@ourworldindata/utils"
 
-export const SiteNavigationTopic = ({ topic }: { topic: EntryMeta }) => {
+export const SiteNavigationTopic = ({ topic }: { topic: TagGraphNode }) => {
     return (
         <li className="SiteNavigationTopic">
             <a href={`/${topic.slug}`} data-track-note="header_navigation">
-                <span className="label">{topic.title}</span>
+                <span className="label">{topic.name}</span>
             </a>
         </li>
     )