Add Related research and writing to multi-dim pages

adminSiteServer/adminRouter.tsx

@@ -360,12 +360,12 @@ getPlainRouteWithROTransaction(
             onlyPublished: false,
         })
         if (mdd) {
-            const renderedPage = await renderMultiDimDataPageFromConfig(
-                trx,
+            const renderedPage = await renderMultiDimDataPageFromConfig({
+                knex: trx,
                 slug,
-                mdd.config,
-                true
-            )
+                config: mdd.config,
+                isPreviewing: true,
+            })
             res.send(renderedPage)
             return
         }

baker/GrapherBaker.tsx

@@ -26,7 +26,7 @@ import {
     getPostIdFromSlug,
     getPostRelatedCharts,
     getRelatedArticles,
-    getRelatedResearchAndWritingForVariable,
+    getRelatedResearchAndWritingForVariables,
 } from "../db/model/Post.js"
 import {
     GrapherInterface,
@@ -200,7 +200,7 @@ export async function renderDataPageV2(
     )
 
     datapageData.relatedResearch =
-        await getRelatedResearchAndWritingForVariable(knex, variableId)
+        await getRelatedResearchAndWritingForVariables(knex, [variableId])
 
     const relatedResearchFilenames = datapageData.relatedResearch
         .map((r) => r.imageUrl)

baker/MultiDimBaker.tsx

@@ -1,6 +1,7 @@
 import fs from "fs-extra"
 import path from "path"
 import {
+    ImageMetadata,
     MultiDimDataPageConfigPreProcessed,
     MultiDimDataPageProps,
     FaqEntryKeyedByGdocIdAndFragmentId,
@@ -11,8 +12,11 @@ import {
     keyBy,
     OwidVariableWithSource,
     pick,
+    uniq,
 } from "@ourworldindata/utils"
 import * as db from "../db/db.js"
+import { getImagesByFilenames } from "../db/model/Image.js"
+import { getRelatedResearchAndWritingForVariables } from "../db/model/Post.js"
 import { renderToHtmlPage } from "./siteRenderers.js"
 import { MultiDimDataPage } from "../site/multiDim/MultiDimDataPage.js"
 import {
@@ -42,10 +46,7 @@ const getRelevantVariableIds = (config: MultiDimDataPageConfigPreProcessed) => {
     return new Set(allIndicatorIds)
 }
 
-const getRelevantVariableMetadata = async (
-    config: MultiDimDataPageConfigPreProcessed
-) => {
-    const variableIds = getRelevantVariableIds(config)
+async function getRelevantVariableMetadata(variableIds: Iterable<number>) {
     const metadata = await pMap(
         variableIds,
         async (id) => {
@@ -114,25 +115,56 @@ const getFaqEntries = async (
     }
 }
 
-export const renderMultiDimDataPageFromConfig = async (
-    knex: db.KnexReadonlyTransaction,
-    slug: string,
-    config: MultiDimDataPageConfigEnriched,
-    isPreviewing: boolean = false
-) => {
+export async function renderMultiDimDataPageFromConfig({
+    knex,
+    slug,
+    config,
+    imageMetadataDictionary,
+    isPreviewing = false,
+}: {
+    knex: db.KnexReadonlyTransaction
+    slug: string
+    config: MultiDimDataPageConfigEnriched
+    imageMetadataDictionary?: Record<string, ImageMetadata>
+    isPreviewing?: boolean
+}) {
     // TAGS
     const tagToSlugMap = await getTagToSlugMap(knex)
     // Only embed the tags that are actually used by the datapage, instead of the complete JSON object with ~240 properties
     const minimalTagToSlugMap = pick(tagToSlugMap, config.topicTags ?? [])
     const pageConfig = MultiDimDataPageConfig.fromObject(config)
+    const variableIds = getRelevantVariableIds(config)
 
     // FAQs
-    const variableMetaDict = await getRelevantVariableMetadata(config)
+    const variableMetaDict = await getRelevantVariableMetadata(variableIds)
     const faqEntries = await getFaqEntries(knex, config, variableMetaDict)
 
     // PRIMARY TOPIC
     const primaryTopic = await getPrimaryTopic(knex, config.topicTags?.[0])
 
+    // Related research
+    const relatedResearchCandidates =
+        variableIds.size > 0
+            ? await getRelatedResearchAndWritingForVariables(knex, [
+                  ...variableIds,
+              ])
+            : []
+
+    const relatedResearchFilenames = uniq(
+        relatedResearchCandidates.map((r) => r.imageUrl).filter((f) => !!f)
+    )
+
+    let imageMetadata: Record<string, ImageMetadata>
+    if (imageMetadataDictionary) {
+        imageMetadata = pick(imageMetadataDictionary, relatedResearchFilenames)
+    } else {
+        const images = await getImagesByFilenames(
+            knex,
+            relatedResearchFilenames
+        )
+        imageMetadata = keyBy(images, "filename")
+    }
+
     const props = {
         baseUrl: BAKED_BASE_URL,
         baseGrapherUrl: BAKED_GRAPHER_URL,
@@ -141,6 +173,8 @@ export const renderMultiDimDataPageFromConfig = async (
         tagToSlugMap: minimalTagToSlugMap,
         faqEntries,
         primaryTopic,
+        relatedResearchCandidates,
+        imageMetadata,
         isPreviewing,
     }
 
@@ -155,7 +189,11 @@ export const renderMultiDimDataPageBySlug = async (
     const dbRow = await getMultiDimDataPageBySlug(knex, slug, { onlyPublished })
     if (!dbRow) throw new Error(`No multi-dim site found for slug: ${slug}`)
 
-    return renderMultiDimDataPageFromConfig(knex, slug, dbRow.config)
+    return renderMultiDimDataPageFromConfig({
+        knex,
+        slug,
+        config: dbRow.config,
+    })
 }
 
 export const renderMultiDimDataPageFromProps = async (
@@ -168,23 +206,32 @@ export const bakeMultiDimDataPage = async (
     knex: db.KnexReadonlyTransaction,
     bakedSiteDir: string,
     slug: string,
-    config: MultiDimDataPageConfigEnriched
+    config: MultiDimDataPageConfigEnriched,
+    imageMetadata: Record<string, ImageMetadata>
 ) => {
-    const renderedHtml = await renderMultiDimDataPageFromConfig(
+    const renderedHtml = await renderMultiDimDataPageFromConfig({
         knex,
         slug,
-        config
-    )
+        config,
+        imageMetadataDictionary: imageMetadata,
+    })
     const outPath = path.join(bakedSiteDir, `grapher/${slug}.html`)
     await fs.writeFile(outPath, renderedHtml)
 }
 
 export const bakeAllMultiDimDataPages = async (
     knex: db.KnexReadonlyTransaction,
-    bakedSiteDir: string
+    bakedSiteDir: string,
+    imageMetadata: Record<string, ImageMetadata>
 ) => {
     const multiDimsBySlug = await getAllMultiDimDataPages(knex)
     for (const [slug, row] of multiDimsBySlug.entries()) {
-        await bakeMultiDimDataPage(knex, bakedSiteDir, slug, row.config)
+        await bakeMultiDimDataPage(
+            knex,
+            bakedSiteDir,
+            slug,
+            row.config,
+            imageMetadata
+        )
     }
 }

baker/SiteBaker.tsx

@@ -181,7 +181,8 @@ function getProgressBarTotal(bakeSteps: BakeStepConfig): number {
         bakeSteps.has("gdocPosts") ||
         bakeSteps.has("gdocTombstones") ||
         bakeSteps.has("dataInsights") ||
-        bakeSteps.has("authors")
+        bakeSteps.has("authors") ||
+        bakeSteps.has("multiDimPages")
     ) {
         total += 9
     }
@@ -845,8 +846,8 @@ export class SiteBaker {
             )
             return
         }
-
-        await bakeAllMultiDimDataPages(knex, this.bakedSiteDir)
+        const { imageMetadata } = await this.getPrefetchedGdocAttachments(knex)
+        await bakeAllMultiDimDataPages(knex, this.bakedSiteDir, imageMetadata)
 
         this.progressBar.tick({ name: "✅ baked multi-dim pages" })
     }

db/model/Image.ts

@@ -14,3 +14,15 @@ export async function getAllImages(
         .select<DbRawImage[]>()
     return images.map(parseImageRow)
 }
+
+export async function getImagesByFilenames(
+    knex: KnexReadonlyTransaction,
+    filenames: string[]
+): Promise<DbEnrichedImage[]> {
+    const images = await knex
+        .table("images")
+        .where("replacedBy", null)
+        .whereIn("filename", filenames)
+        .select<DbRawImage[]>()
+    return images.map(parseImageRow)
+}

db/model/Post.ts

@@ -513,9 +513,9 @@ export interface RelatedResearchQueryResult {
     tags: string
 }
 
-export const getRelatedResearchAndWritingForVariable = async (
+export const getRelatedResearchAndWritingForVariables = async (
     knex: db.KnexReadonlyTransaction,
-    variableId: number
+    variableIds: Iterable<number>
 ): Promise<DataPageRelatedResearch[]> => {
     const wp_posts: RelatedResearchQueryResult[] = await db.knexRaw(
         knex,
@@ -563,7 +563,7 @@ export const getRelatedResearchAndWritingForVariable = async (
                 -- this means that only the links that are of the iframe kind will be kept - normal a href style links will
                 -- be disregarded
                 AND componentType = 'src'
-                AND cd.variableId = ?
+                AND cd.variableId IN (?)
                 AND cd.property IN ('x', 'y') -- ignore cases where the indicator is size, color etc
                 AND p.status = 'publish' -- only use published wp posts
                 AND p.type != 'wp_block'
@@ -575,7 +575,7 @@ export const getRelatedResearchAndWritingForVariable = async (
                 -- but that replace an old wordpress page
 
             `,
-        [variableId]
+        [variableIds]
     )
 
     const gdocs_posts: RelatedResearchQueryResult[] = await db.knexRaw(
@@ -613,11 +613,11 @@ export const getRelatedResearchAndWritingForVariable = async (
         WHERE
             pl.linkType = 'grapher'
             AND componentType = 'chart' -- this filters out links in tags and keeps only embedded charts
-            AND cd.variableId = ?
+            AND cd.variableId IN (?)
             AND cd.property IN ('x', 'y') -- ignore cases where the indicator is size, color etc
             AND p.published = 1
             AND p.type != 'fragment'`,
-        [variableId]
+        [variableIds]
     )
 
     const combined = [...wp_posts, ...gdocs_posts]
@@ -642,7 +642,7 @@ export const getRelatedResearchAndWritingForVariable = async (
     // the queries above use distinct but because of the information we pull in if the same piece of research
     // uses different charts that all use a single indicator we would get duplicates for the post to link to so
     // here we deduplicate by url. The first item is retained by uniqBy, latter ones are discarded.
-    return uniqBy(allSortedRelatedResearch, "url")
+    return uniqBy(allSortedRelatedResearch, "url").slice(0, 20)
 }
 
 export const getLatestWorkByAuthor = async (

packages/@ourworldindata/types/src/siteTypes/MultiDimDataPage.ts

@@ -1,5 +1,6 @@
 import { OwidEnrichedGdocBlock } from "../gdocTypes/ArchieMlComponents.js"
-import { PrimaryTopic } from "../gdocTypes/Datapage.js"
+import { DataPageRelatedResearch, PrimaryTopic } from "../gdocTypes/Datapage.js"
+import { ImageMetadata } from "../gdocTypes/Image.js"
 import { GrapherInterface } from "../grapherTypes/GrapherTypes.js"
 import {
     IndicatorTitleWithFragments,
@@ -115,7 +116,8 @@ export interface MultiDimDataPageProps {
     tagToSlugMap?: Record<string, string>
     faqEntries?: FaqEntryKeyedByGdocIdAndFragmentId
     primaryTopic?: PrimaryTopic
-
+    relatedResearchCandidates: DataPageRelatedResearch[]
+    imageMetadata: Record<string, ImageMetadata>
     initialQueryStr?: string
     isPreviewing?: boolean
 }

site/DataPageResearchAndWriting.tsx

@@ -0,0 +1,66 @@
+import {
+    DataPageRelatedResearch,
+    DEFAULT_THUMBNAIL_FILENAME,
+} from "@ourworldindata/types"
+import { formatAuthors } from "@ourworldindata/utils"
+import { BAKED_BASE_URL } from "../settings/clientSettings.js"
+import Image from "./gdocs/components/Image.js"
+
+export default function DataPageResearchAndWriting({
+    relatedResearch,
+}: {
+    relatedResearch: DataPageRelatedResearch[]
+}) {
+    return (
+        <div className="section-wrapper grid">
+            <h2
+                className="related-research__title span-cols-3 span-lg-cols-12"
+                id="research-and-writing"
+            >
+                Related research and writing
+            </h2>
+            <div className="related-research__items grid grid-cols-9 grid-lg-cols-12 span-cols-9 span-lg-cols-12">
+                {relatedResearch.map((research) => (
+                    <a
+                        href={research.url}
+                        key={research.url}
+                        className="related-research__item grid grid-cols-4 grid-lg-cols-6 grid-sm-cols-12 span-cols-4 span-lg-cols-6 span-sm-cols-12"
+                    >
+                        <Thumbnail filename={research.imageUrl} />
+                        <div className="span-cols-3 span-lg-cols-4 span-sm-cols-9">
+                            <h3 className="related-article__title">
+                                {research.title}
+                            </h3>
+                            <div className="related-article__authors body-3-medium-italic">
+                                {research.authors &&
+                                    research.authors.length &&
+                                    formatAuthors({
+                                        authors: research.authors,
+                                    })}
+                            </div>
+                        </div>
+                    </a>
+                ))}
+            </div>
+        </div>
+    )
+}
+
+function Thumbnail({ filename }: { filename: string }) {
+    if (!filename) {
+        return (
+            <img
+                className="span-lg-cols-2 span-sm-cols-3"
+                src={`${BAKED_BASE_URL}/${DEFAULT_THUMBNAIL_FILENAME}`}
+            />
+        )
+    }
+    return (
+        <Image
+            className="span-lg-cols-2 span-sm-cols-3"
+            containerType="thumbnail"
+            filename={filename}
+            shouldLightbox={false}
+        />
+    )
+}