From 3f3c97e4346f0c00c5f123befad7b30fce083920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ra=C4=8D=C3=A1k?= Date: Wed, 22 Jan 2025 11:52:54 +0100 Subject: [PATCH] Add Related research and writing to multi-dim pages --- adminSiteServer/adminRouter.tsx | 10 +- baker/GrapherBaker.tsx | 4 +- baker/MultiDimBaker.tsx | 83 ++++-- baker/SiteBaker.tsx | 7 +- db/model/Image.ts | 12 + db/model/Post.ts | 14 +- .../types/src/siteTypes/MultiDimDataPage.ts | 6 +- site/DataPageResearchAndWriting.tsx | 66 +++++ site/DataPageV2Content.tsx | 102 +------ site/dataPage.ts | 28 ++ site/multiDim/MultiDimDataPage.tsx | 4 + site/multiDim/MultiDimDataPageContent.tsx | 250 ++++++++---------- 12 files changed, 314 insertions(+), 272 deletions(-) create mode 100644 site/DataPageResearchAndWriting.tsx create mode 100644 site/dataPage.ts diff --git a/adminSiteServer/adminRouter.tsx b/adminSiteServer/adminRouter.tsx index f239b00ea5..7d7a52f41b 100644 --- a/adminSiteServer/adminRouter.tsx +++ b/adminSiteServer/adminRouter.tsx @@ -360,12 +360,12 @@ getPlainRouteWithROTransaction( onlyPublished: false, }) if (mdd) { - const renderedPage = await renderMultiDimDataPageFromConfig( - trx, + const renderedPage = await renderMultiDimDataPageFromConfig({ + knex: trx, slug, - mdd.config, - true - ) + config: mdd.config, + isPreviewing: true, + }) res.send(renderedPage) return } diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx index 2efc3a6c52..74f3fa4eb8 100644 --- a/baker/GrapherBaker.tsx +++ b/baker/GrapherBaker.tsx @@ -26,7 +26,7 @@ import { getPostIdFromSlug, getPostRelatedCharts, getRelatedArticles, - getRelatedResearchAndWritingForVariable, + getRelatedResearchAndWritingForVariables, } from "../db/model/Post.js" import { GrapherInterface, @@ -200,7 +200,7 @@ export async function renderDataPageV2( ) datapageData.relatedResearch = - await getRelatedResearchAndWritingForVariable(knex, variableId) + await getRelatedResearchAndWritingForVariables(knex, [variableId]) const relatedResearchFilenames = datapageData.relatedResearch .map((r) => r.imageUrl) diff --git a/baker/MultiDimBaker.tsx b/baker/MultiDimBaker.tsx index c76898ebf7..7aaf2a5ec5 100644 --- a/baker/MultiDimBaker.tsx +++ b/baker/MultiDimBaker.tsx @@ -1,6 +1,7 @@ import fs from "fs-extra" import path from "path" import { + ImageMetadata, MultiDimDataPageConfigPreProcessed, MultiDimDataPageProps, FaqEntryKeyedByGdocIdAndFragmentId, @@ -11,8 +12,11 @@ import { keyBy, OwidVariableWithSource, pick, + uniq, } from "@ourworldindata/utils" import * as db from "../db/db.js" +import { getImagesByFilenames } from "../db/model/Image.js" +import { getRelatedResearchAndWritingForVariables } from "../db/model/Post.js" import { renderToHtmlPage } from "./siteRenderers.js" import { MultiDimDataPage } from "../site/multiDim/MultiDimDataPage.js" import { @@ -42,10 +46,7 @@ const getRelevantVariableIds = (config: MultiDimDataPageConfigPreProcessed) => { return new Set(allIndicatorIds) } -const getRelevantVariableMetadata = async ( - config: MultiDimDataPageConfigPreProcessed -) => { - const variableIds = getRelevantVariableIds(config) +async function getRelevantVariableMetadata(variableIds: Iterable) { const metadata = await pMap( variableIds, async (id) => { @@ -114,25 +115,56 @@ const getFaqEntries = async ( } } -export const renderMultiDimDataPageFromConfig = async ( - knex: db.KnexReadonlyTransaction, - slug: string, - config: MultiDimDataPageConfigEnriched, - isPreviewing: boolean = false -) => { +export async function renderMultiDimDataPageFromConfig({ + knex, + slug, + config, + imageMetadataDictionary, + isPreviewing = false, +}: { + knex: db.KnexReadonlyTransaction + slug: string + config: MultiDimDataPageConfigEnriched + imageMetadataDictionary?: Record + isPreviewing?: boolean +}) { // TAGS const tagToSlugMap = await getTagToSlugMap(knex) // Only embed the tags that are actually used by the datapage, instead of the complete JSON object with ~240 properties const minimalTagToSlugMap = pick(tagToSlugMap, config.topicTags ?? []) const pageConfig = MultiDimDataPageConfig.fromObject(config) + const variableIds = getRelevantVariableIds(config) // FAQs - const variableMetaDict = await getRelevantVariableMetadata(config) + const variableMetaDict = await getRelevantVariableMetadata(variableIds) const faqEntries = await getFaqEntries(knex, config, variableMetaDict) // PRIMARY TOPIC const primaryTopic = await getPrimaryTopic(knex, config.topicTags?.[0]) + // Related research + const relatedResearchCandidates = + variableIds.size > 0 + ? await getRelatedResearchAndWritingForVariables(knex, [ + ...variableIds, + ]) + : [] + + const relatedResearchFilenames = uniq( + relatedResearchCandidates.map((r) => r.imageUrl).filter((f) => !!f) + ) + + let imageMetadata: Record + if (imageMetadataDictionary) { + imageMetadata = pick(imageMetadataDictionary, relatedResearchFilenames) + } else { + const images = await getImagesByFilenames( + knex, + relatedResearchFilenames + ) + imageMetadata = keyBy(images, "filename") + } + const props = { baseUrl: BAKED_BASE_URL, baseGrapherUrl: BAKED_GRAPHER_URL, @@ -141,6 +173,8 @@ export const renderMultiDimDataPageFromConfig = async ( tagToSlugMap: minimalTagToSlugMap, faqEntries, primaryTopic, + relatedResearchCandidates, + imageMetadata, isPreviewing, } @@ -155,7 +189,11 @@ export const renderMultiDimDataPageBySlug = async ( const dbRow = await getMultiDimDataPageBySlug(knex, slug, { onlyPublished }) if (!dbRow) throw new Error(`No multi-dim site found for slug: ${slug}`) - return renderMultiDimDataPageFromConfig(knex, slug, dbRow.config) + return renderMultiDimDataPageFromConfig({ + knex, + slug, + config: dbRow.config, + }) } export const renderMultiDimDataPageFromProps = async ( @@ -168,23 +206,32 @@ export const bakeMultiDimDataPage = async ( knex: db.KnexReadonlyTransaction, bakedSiteDir: string, slug: string, - config: MultiDimDataPageConfigEnriched + config: MultiDimDataPageConfigEnriched, + imageMetadata: Record ) => { - const renderedHtml = await renderMultiDimDataPageFromConfig( + const renderedHtml = await renderMultiDimDataPageFromConfig({ knex, slug, - config - ) + config, + imageMetadataDictionary: imageMetadata, + }) const outPath = path.join(bakedSiteDir, `grapher/${slug}.html`) await fs.writeFile(outPath, renderedHtml) } export const bakeAllMultiDimDataPages = async ( knex: db.KnexReadonlyTransaction, - bakedSiteDir: string + bakedSiteDir: string, + imageMetadata: Record ) => { const multiDimsBySlug = await getAllMultiDimDataPages(knex) for (const [slug, row] of multiDimsBySlug.entries()) { - await bakeMultiDimDataPage(knex, bakedSiteDir, slug, row.config) + await bakeMultiDimDataPage( + knex, + bakedSiteDir, + slug, + row.config, + imageMetadata + ) } } diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index d6e4579abe..bb395c4aff 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -181,7 +181,8 @@ function getProgressBarTotal(bakeSteps: BakeStepConfig): number { bakeSteps.has("gdocPosts") || bakeSteps.has("gdocTombstones") || bakeSteps.has("dataInsights") || - bakeSteps.has("authors") + bakeSteps.has("authors") || + bakeSteps.has("multiDimPages") ) { total += 9 } @@ -845,8 +846,8 @@ export class SiteBaker { ) return } - - await bakeAllMultiDimDataPages(knex, this.bakedSiteDir) + const { imageMetadata } = await this.getPrefetchedGdocAttachments(knex) + await bakeAllMultiDimDataPages(knex, this.bakedSiteDir, imageMetadata) this.progressBar.tick({ name: "✅ baked multi-dim pages" }) } diff --git a/db/model/Image.ts b/db/model/Image.ts index e4e241d35a..505449e88c 100644 --- a/db/model/Image.ts +++ b/db/model/Image.ts @@ -14,3 +14,15 @@ export async function getAllImages( .select() return images.map(parseImageRow) } + +export async function getImagesByFilenames( + knex: KnexReadonlyTransaction, + filenames: string[] +): Promise { + const images = await knex + .table("images") + .where("replacedBy", null) + .whereIn("filename", filenames) + .select() + return images.map(parseImageRow) +} diff --git a/db/model/Post.ts b/db/model/Post.ts index d8a7788a2a..df34cdc905 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -513,9 +513,9 @@ export interface RelatedResearchQueryResult { tags: string } -export const getRelatedResearchAndWritingForVariable = async ( +export const getRelatedResearchAndWritingForVariables = async ( knex: db.KnexReadonlyTransaction, - variableId: number + variableIds: Iterable ): Promise => { const wp_posts: RelatedResearchQueryResult[] = await db.knexRaw( knex, @@ -563,7 +563,7 @@ export const getRelatedResearchAndWritingForVariable = async ( -- this means that only the links that are of the iframe kind will be kept - normal a href style links will -- be disregarded AND componentType = 'src' - AND cd.variableId = ? + AND cd.variableId IN (?) AND cd.property IN ('x', 'y') -- ignore cases where the indicator is size, color etc AND p.status = 'publish' -- only use published wp posts AND p.type != 'wp_block' @@ -575,7 +575,7 @@ export const getRelatedResearchAndWritingForVariable = async ( -- but that replace an old wordpress page `, - [variableId] + [variableIds] ) const gdocs_posts: RelatedResearchQueryResult[] = await db.knexRaw( @@ -613,11 +613,11 @@ export const getRelatedResearchAndWritingForVariable = async ( WHERE pl.linkType = 'grapher' AND componentType = 'chart' -- this filters out links in tags and keeps only embedded charts - AND cd.variableId = ? + AND cd.variableId IN (?) AND cd.property IN ('x', 'y') -- ignore cases where the indicator is size, color etc AND p.published = 1 AND p.type != 'fragment'`, - [variableId] + [variableIds] ) const combined = [...wp_posts, ...gdocs_posts] @@ -642,7 +642,7 @@ export const getRelatedResearchAndWritingForVariable = async ( // the queries above use distinct but because of the information we pull in if the same piece of research // uses different charts that all use a single indicator we would get duplicates for the post to link to so // here we deduplicate by url. The first item is retained by uniqBy, latter ones are discarded. - return uniqBy(allSortedRelatedResearch, "url") + return uniqBy(allSortedRelatedResearch, "url").slice(0, 20) } export const getLatestWorkByAuthor = async ( diff --git a/packages/@ourworldindata/types/src/siteTypes/MultiDimDataPage.ts b/packages/@ourworldindata/types/src/siteTypes/MultiDimDataPage.ts index cd00af8bbf..6819bea4df 100644 --- a/packages/@ourworldindata/types/src/siteTypes/MultiDimDataPage.ts +++ b/packages/@ourworldindata/types/src/siteTypes/MultiDimDataPage.ts @@ -1,5 +1,6 @@ import { OwidEnrichedGdocBlock } from "../gdocTypes/ArchieMlComponents.js" -import { PrimaryTopic } from "../gdocTypes/Datapage.js" +import { DataPageRelatedResearch, PrimaryTopic } from "../gdocTypes/Datapage.js" +import { ImageMetadata } from "../gdocTypes/Image.js" import { GrapherInterface } from "../grapherTypes/GrapherTypes.js" import { IndicatorTitleWithFragments, @@ -115,7 +116,8 @@ export interface MultiDimDataPageProps { tagToSlugMap?: Record faqEntries?: FaqEntryKeyedByGdocIdAndFragmentId primaryTopic?: PrimaryTopic - + relatedResearchCandidates: DataPageRelatedResearch[] + imageMetadata: Record initialQueryStr?: string isPreviewing?: boolean } diff --git a/site/DataPageResearchAndWriting.tsx b/site/DataPageResearchAndWriting.tsx new file mode 100644 index 0000000000..53d2c9abee --- /dev/null +++ b/site/DataPageResearchAndWriting.tsx @@ -0,0 +1,66 @@ +import { + DataPageRelatedResearch, + DEFAULT_THUMBNAIL_FILENAME, +} from "@ourworldindata/types" +import { formatAuthors } from "@ourworldindata/utils" +import { BAKED_BASE_URL } from "../settings/clientSettings.js" +import Image from "./gdocs/components/Image.js" + +export default function DataPageResearchAndWriting({ + relatedResearch, +}: { + relatedResearch: DataPageRelatedResearch[] +}) { + return ( + + ) +} + +function Thumbnail({ filename }: { filename: string }) { + if (!filename) { + return ( + + ) + } + return ( + + ) +} diff --git a/site/DataPageV2Content.tsx b/site/DataPageV2Content.tsx index 673dbc7a39..a856513207 100644 --- a/site/DataPageV2Content.tsx +++ b/site/DataPageV2Content.tsx @@ -8,21 +8,18 @@ import { GrapherWithFallback } from "./GrapherWithFallback.js" import { RelatedCharts } from "./blocks/RelatedCharts.js" import { DataPageV2ContentFields, - formatAuthors, - intersection, GrapherInterface, joinTitleFragments, ImageMetadata, - DEFAULT_THUMBNAIL_FILENAME, } from "@ourworldindata/utils" import { DocumentContext } from "./gdocs/DocumentContext.js" import { AttachmentsContext } from "./gdocs/AttachmentsContext.js" import StickyNav from "./blocks/StickyNav.js" -import { BAKED_BASE_URL } from "../settings/clientSettings.js" -import Image from "./gdocs/components/Image.js" import AboutThisData from "./AboutThisData.js" +import DataPageResearchAndWriting from "./DataPageResearchAndWriting.js" import MetadataSection from "./MetadataSection.js" import TopicTags from "./TopicTags.js" +import { processRelatedResearch } from "./dataPage.js" declare global { interface Window { @@ -32,34 +29,6 @@ declare global { } export const OWID_DATAPAGE_CONTENT_ROOT_ID = "owid-datapageJson-root" -// We still have topic pages on WordPress, whose featured images are specified as absolute URLs which this component handles -// Once we've migrated all WP topic pages to gdocs, we'll be able to remove this component and just use -const DatapageResearchThumbnail = ({ - urlOrFilename, -}: { - urlOrFilename: string | undefined | null -}) => { - if (!urlOrFilename) { - urlOrFilename = `${BAKED_BASE_URL}/${DEFAULT_THUMBNAIL_FILENAME}` - } - if (urlOrFilename.startsWith("http")) { - return ( - - ) - } - return ( - - ) -} - export const DataPageV2Content = ({ datapageData, grapherConfig, @@ -111,29 +80,10 @@ export const DataPageV2Content = ({ { text: "Reuse This Work", target: "#" + REUSE_THIS_WORK_SECTION_ID }, ] - const relatedResearchCandidates = datapageData.relatedResearch - const relatedResearch = - relatedResearchCandidates.length > 3 && - datapageData.topicTagsLinks?.length - ? relatedResearchCandidates.filter((research) => { - const shared = intersection( - research.tags, - datapageData.topicTagsLinks ?? [] - ) - return shared.length > 0 - }) - : relatedResearchCandidates - for (const item of relatedResearch) { - // TODO: these are workarounds to not link to the (not really existing) template pages for energy or co2 - // country profiles but instead to the topic page at the country selector. - if (item.url === "/co2-country-profile") - item.url = - "/co2-and-greenhouse-gas-emissions#co2-and-greenhouse-gas-emissions-country-profiles" - else if (item.url === "/energy-country-profile") - item.url = "/energy#country-profiles" - else if (item.url === "/coronavirus-country-profile") - item.url = "/coronavirus#coronavirus-country-profiles" - } + const relatedResearch = processRelatedResearch( + datapageData.relatedResearch, + datapageData.topicTagsLinks ?? [] + ) return (
{relatedResearch && relatedResearch.length > 0 && ( - + )} {datapageData.allCharts && datapageData.allCharts.length > 0 ? ( diff --git a/site/dataPage.ts b/site/dataPage.ts new file mode 100644 index 0000000000..226306dc83 --- /dev/null +++ b/site/dataPage.ts @@ -0,0 +1,28 @@ +import { DataPageRelatedResearch, intersection } from "@ourworldindata/utils" + +export function processRelatedResearch( + candidates: DataPageRelatedResearch[], + topicTags: string[] +) { + let relatedResearch + if (candidates.length > 3 && topicTags.length > 0) { + relatedResearch = candidates.filter((research) => { + const shared = intersection(research.tags, topicTags) + return shared.length > 0 + }) + } else { + relatedResearch = [...candidates] + } + for (const item of relatedResearch) { + // TODO: these are workarounds to not link to the (not really existing) template pages for energy or co2 + // country profiles but instead to the topic page at the country selector. + if (item.url === "/co2-country-profile") + item.url = + "/co2-and-greenhouse-gas-emissions#co2-and-greenhouse-gas-emissions-country-profiles" + else if (item.url === "/energy-country-profile") + item.url = "/energy#country-profiles" + else if (item.url === "/coronavirus-country-profile") + item.url = "/coronavirus#coronavirus-country-profiles" + } + return relatedResearch +} diff --git a/site/multiDim/MultiDimDataPage.tsx b/site/multiDim/MultiDimDataPage.tsx index b056d786bc..5cfcdd12db 100644 --- a/site/multiDim/MultiDimDataPage.tsx +++ b/site/multiDim/MultiDimDataPage.tsx @@ -16,6 +16,8 @@ export function MultiDimDataPage({ tagToSlugMap, faqEntries, primaryTopic, + relatedResearchCandidates, + imageMetadata, initialQueryStr, isPreviewing, }: MultiDimDataPageProps) { @@ -26,6 +28,8 @@ export function MultiDimDataPage({ configObj, faqEntries, primaryTopic, + relatedResearchCandidates, + imageMetadata, tagToSlugMap, initialQueryStr, } diff --git a/site/multiDim/MultiDimDataPageContent.tsx b/site/multiDim/MultiDimDataPageContent.tsx index b79d26e945..8d202ecd34 100644 --- a/site/multiDim/MultiDimDataPageContent.tsx +++ b/site/multiDim/MultiDimDataPageContent.tsx @@ -28,7 +28,9 @@ import { import cx from "classnames" import { ADMIN_BASE_URL, DATA_API_URL } from "../../settings/clientSettings.js" import { + DataPageRelatedResearch, FaqEntryKeyedByGdocIdAndFragmentId, + ImageMetadata, MultiDimDataPageConfigEnriched, MultiDimDimensionChoices, PrimaryTopic, @@ -39,6 +41,9 @@ import TopicTags from "../TopicTags.js" import MetadataSection from "../MetadataSection.js" import { useElementBounds, useMobxStateToReactState } from "../hooks.js" import { MultiDimSettingsPanel } from "./MultiDimDataPageSettingsPanel.js" +import { processRelatedResearch } from "../dataPage.js" +import DataPageResearchAndWriting from "../DataPageResearchAndWriting.js" +import { AttachmentsContext } from "../gdocs/AttachmentsContext.js" declare global { interface Window { @@ -207,7 +212,9 @@ export type MultiDimDataPageContentProps = { tagToSlugMap?: Record faqEntries?: FaqEntryKeyedByGdocIdAndFragmentId primaryTopic?: PrimaryTopic + relatedResearchCandidates: DataPageRelatedResearch[] initialQueryStr?: string + imageMetadata: Record isPreviewing?: boolean } @@ -219,8 +226,9 @@ export const MultiDimDataPageContent = ({ isPreviewing, faqEntries, primaryTopic, + relatedResearchCandidates, tagToSlugMap, - // imageMetadata, + imageMetadata, initialQueryStr, }: MultiDimDataPageContentProps) => { const grapherFigureRef = useRef(null) @@ -315,29 +323,14 @@ export const MultiDimDataPageContent = ({ const hasTopicTags = !!config.config.topicTags?.length - // TODO - // const relatedResearchCandidates = varDatapageData?.relatedResearch ?? [] - // const relatedResearch = - // relatedResearchCandidates.length > 3 && config.config.topicTags?.length - // ? relatedResearchCandidates.filter((research) => { - // const shared = intersection( - // research.tags, - // config.config.topicTags ?? [] - // ) - // return shared.length > 0 - // }) - // : relatedResearchCandidates - // for (const item of relatedResearch) { - // // TODO: these are workarounds to not link to the (not really existing) template pages for energy or co2 - // // country profiles but instead to the topic page at the country selector. - // if (item.url === "/co2-country-profile") - // item.url = - // "/co2-and-greenhouse-gas-emissions#co2-and-greenhouse-gas-emissions-country-profiles" - // else if (item.url === "/energy-country-profile") - // item.url = "/energy#country-profiles" - // else if (item.url === "/coronavirus-country-profile") - // item.url = "/coronavirus#coronavirus-country-profiles" - // } + const relatedResearch = useMemo( + () => + processRelatedResearch( + relatedResearchCandidates ?? [], + config.config.topicTags ?? [] + ), + [relatedResearchCandidates, config.config.topicTags] + ) const faqEntriesForView = useMemo(() => { return compact( @@ -348,132 +341,105 @@ export const MultiDimDataPageContent = ({ }, [varDatapageData?.faqs, faqEntries]) return ( -
-
-
-
-
Data
-

- {config.config.title.title} -

-
{titleFragments}
-
- {hasTopicTags && tagToSlugMap && ( - - )} -
+
+
+
+
+
Data
+

+ {config.config.title.title} +

+
+ {titleFragments} +
+
+ {hasTopicTags && tagToSlugMap && ( + )} - > - -
-
-
- -
-
-
-
- + -
+
- {varDatapageData && ( - - )}
-
- {/*
- {relatedResearch && relatedResearch.length > 0 && ( -
-

+
+
- Related research and writing -

- + {varDatapageData && ( + + )}
- )} - {varDatapageData?.allCharts && - varDatapageData?.allCharts.length > 0 ? ( -
-

- Explore charts that include this data -

-
- +
+
+ {relatedResearch && relatedResearch.length > 0 && ( + -
+ )}
- ) : null} -
*/} - {varDatapageData && ( - - )} -
+
+ {varDatapageData && ( + + )} +
+ ) }