formatWordpressPost.tsx

import cheerio from "cheerio"
import urlSlug from "url-slug"
import ReactDOMServer from "react-dom/server.js"
import { HTTPS_ONLY } from "../settings/serverSettings.js"
import { FormattingOptions, GRAPHER_PREVIEW_CLASS } from "@ourworldindata/types"
import { FormattedPost, FullPost, TocHeading, Url } from "@ourworldindata/utils"
import { parseKeyValueArgs } from "../serverUtils/wordpressUtils.js"
import { Footnote } from "../site/Footnote.js"
import { PROMINENT_LINK_CLASSNAME } from "../site/blocks/ProminentLink.js"
import { DataToken } from "../site/DataToken.js"
import { DEEP_LINK_CLASS, formatImages } from "./formatting.js"
import { replaceIframesWithExplorerRedirectsInWordPressPost } from "./replaceExplorerRedirects.js"
import { EXPLORERS_ROUTE_FOLDER } from "@ourworldindata/explorer"
import { ADDITIONAL_INFORMATION_CLASS_NAME } from "../site/blocks/AdditionalInformation.js"
import { renderAdditionalInformation } from "../site/blocks/renderAdditionalInformation.js"
import { renderHelp } from "../site/blocks/renderHelp.js"
import { renderCodeSnippets } from "@ourworldindata/components"
import { formatUrls, getBodyHtml } from "../site/formatting.js"
import { renderProminentLinks } from "./siteRenderers.js"
import { RELATED_CHARTS_CLASS_NAME } from "../site/blocks/RelatedCharts.js"
import { KnexReadonlyTransaction } from "../db/db.js"
import {
    EXPLORER_DYNAMIC_THUMBNAIL_URL,
    GRAPHER_DYNAMIC_THUMBNAIL_URL,
} from "../settings/clientSettings.js"

export const formatWordpressPost = async (
    post: FullPost,
    formattingOptions: FormattingOptions,
    knex: KnexReadonlyTransaction
): Promise<FormattedPost> => {
    let html = post.content

    // Standardize urls
    html = formatUrls(html)

    // Strip comments
    html = html.replace(/<!--[^>]+-->/g, "")

    // Footnotes
    const footnotes: string[] = []
    html = html.replace(/{ref}([\s\S]*?){\/ref}/gm, (_, footnote) => {
        if (formattingOptions.footnotes) {
            footnotes.push(footnote)
            const i = footnotes.length
            const href = `#note-${i}`

            return ReactDOMServer.renderToStaticMarkup(
                <a id={`ref-${i}`} className="ref" href={href}>
                    <Footnote index={i} />
                </a>
            )
        } else {
            return ""
        }
    })

    // The only two supported ones here are: {{LastUpdated timestampUrl:...}} and {{FullWidthRawHtml url:...}}
    const dataTokenRegex = /{{\s*([a-zA-Z]+)\s*(.+?)\s*}}/g

    html = html.replace(
        dataTokenRegex,
        (_, token, dataTokenConfigurationString) => {
            return ReactDOMServer.renderToString(
                <DataToken
                    token={token}
                    {...parseKeyValueArgs(dataTokenConfigurationString)}
                />
            )
        }
    )

    // Give "Add country" text (and variations) the appearance of "+ Add Country" chart control
    html = html.replace(
        /(\+ )?[a|A]dd [c|C]ountry/g,
        `<span class="add-country">
            <span class="icon">
                <svg viewBox="0 0 512 512">
                    <path fill="currentColor" d="M410.3 231l11.3-11.3-33.9-33.9-62.1-62.1L291.7 89.8l-11.3 11.3-22.6 22.6L58.6 322.9c-10.4 10.4-18 23.3-22.2 37.4L1 480.7c-2.5 8.4-.2 17.5 6.1 23.7s15.3 8.5 23.7 6.1l120.3-35.4c14.1-4.2 27-11.8 37.4-22.2L387.7 253.7 410.3 231zM160 399.4l-9.1 22.7c-4 3.1-8.5 5.4-13.3 6.9L59.4 452l23-78.1c1.4-4.9 3.8-9.4 6.9-13.3l22.7-9.1v32c0 8.8 7.2 16 16 16h32zM362.7 18.7L348.3 33.2 325.7 55.8 314.3 67.1l33.9 33.9 62.1 62.1 33.9 33.9 11.3-11.3 22.6-22.6 14.5-14.5c25-25 25-65.5 0-90.5L453.3 18.7c-25-25-65.5-25-90.5 0zm-47.4 168l-144 144c-6.2 6.2-16.4 6.2-22.6 0s-6.2-16.4 0-22.6l144-144c6.2-6.2 16.4-6.2 22.6 0s6.2 16.4 0 22.6z"></path>
                </svg>
            </span>
            Edit countries and regions
        </span>`
    )

    const cheerioEl = cheerio.load(html)

    // SSR rendering of Gutenberg blocks, before hydration on client
    //
    // - Note: any post-processing on these blocks runs the risk of hydration
    //   discrepancies. E.g. the ToC post-processing further below add an "id"
    //   attribute to eligible heading tags. In an unbridled version of that
    //   script, the AdditionalInformation block title (h3) would be altered and
    //   receive an "id" attribute (<h3 id="some-title">). When this block is
    //   then hydrated on the client, the "id" attribute is missing, since it
    //   wasn't generated by the isomorphic React component code (but rather
    //   added by the external ToC post-processing code). So from React's
    //   perspective, the server rendered version is different from the client
    //   one, hence the discrepancy.
    renderAdditionalInformation(cheerioEl)
    renderCodeSnippets(cheerioEl)
    renderHelp(cheerioEl)
    await renderProminentLinks(cheerioEl, post.id, knex)

    // Extract inline styling
    let style
    const $style = cheerioEl("style")
    if ($style.length) {
        style = $style
            .toArray()
            .map((el) => cheerioEl(el).html() || "")
            .reduce((prev, curr) => prev + curr)
        $style.remove()
    }

    // Extract blog info content
    let info
    const $info = cheerioEl(".blog-info")
    if ($info.length) {
        info = $info.html() ?? undefined
        $info.remove()
    }

    // Extract last updated
    let lastUpdated
    const $lastUpdated = cheerioEl(".wp-block-last-updated")
    if ($lastUpdated.length) {
        lastUpdated = $lastUpdated.find("p").first().html() ?? undefined
        $lastUpdated.remove()
    }

    // Extract page supertitle
    let supertitle
    const $supertitle = cheerioEl(".wp-block-owid-supertitle")
    if ($supertitle.length) {
        supertitle = $supertitle.text()
        $supertitle.remove()
    }

    // Extract page byline
    let byline
    const $byline = cheerioEl(".wp-block-owid-byline")
    if ($byline.length) {
        byline = $byline.html() ?? undefined
        $byline.remove()
    }

    // Replace URLs pointing to Explorer redirect URLs with the destination URLs
    replaceIframesWithExplorerRedirectsInWordPressPost(cheerioEl)

    const grapherIframes = cheerioEl("iframe")
        .toArray()
        .filter((el) => (el.attribs["src"] || "").match(/\/grapher\//))

    for (const el of grapherIframes) {
        const $el = cheerioEl(el)
        const src = el.attribs["src"].trim()
        const url = Url.fromURL(src)
        const output = `
                <figure data-grapher-src="${src}" class="${GRAPHER_PREVIEW_CLASS}">
                    <a href="${src}" target="_blank">
                    <div>
                        <img
                          src="${GRAPHER_DYNAMIC_THUMBNAIL_URL}/${url.slug}.png${url.queryStr}"
                          width="850"
                          height="600"
                          loading="lazy"
                          data-no-lightbox />
                    </div>
                </figure>`
        if (el.parent.tagName === "p") {
            // We are about to replace <iframe> with <figure>. However, there cannot be <figure> within <p>,
            // so we are lifting the <figure> out.
            // Where does this markup  come from? Historically, wpautop wrapped <iframe> in <p>. Some non-Gutengerg
            // posts will still show that, until they are converted. As a reminder, wpautop is not being used
            // on the overall post content anymore, neither on the Wordpress side nor on the grapher side (through
            // the wpautop npm package), but its effects are still "present" after the result of wpautop were committed
            // to the DB during a one-time refactoring session.
            // <p><iframe></iframe></p>  -->  <p></p><figure></figure>
            const $p = $el.parent()
            $p.after(output)
            $el.remove()
        } else if (el.parent.tagName === "figure") {
            // Support for <iframe> wrapped in <figure>
            // <figure> automatically added by Gutenberg on copy / paste <iframe>
            // Lifting up <iframe> out of <figure>, before it becomes a <figure> itself.
            // <figure><iframe></iframe></figure>  -->  <figure></figure>
            const $figure = $el.parent()
            $figure.after(output)
            $figure.remove()
        } else {
            // No lifting up otherwise, just replacing <iframe> with <figure>
            // <iframe></iframe>  -->  <figure></figure>
            $el.after(output)
            $el.remove()
        }
    }

    // Replace explorer iframes with iframeless embed
    const explorerIframes = cheerioEl("iframe")
        .toArray()
        .filter((el) =>
            (el.attribs["src"] || "").includes(`/${EXPLORERS_ROUTE_FOLDER}/`)
        )
    for (const el of explorerIframes) {
        const $el = cheerioEl(el)
        const src = el.attribs["src"].trim()
        const url = Url.fromURL(src)
        // set a default style if none exists on the existing iframe
        const style = el.attribs["style"] || "width: 100%; height: 600px;"
        const cssClass = el.attribs["class"]
        const $figure = cheerioEl(
            ReactDOMServer.renderToStaticMarkup(
                <figure data-explorer-src={src} className={cssClass}>
                    <img
                        src={`${EXPLORER_DYNAMIC_THUMBNAIL_URL}/${url.slug}.png${url.queryStr}`}
                    />
                </figure>
            )
        )
        $figure.attr("style", style)
        $el.after($figure)
        $el.remove()
    }

    // Any remaining iframes
    for (const iframe of cheerioEl("iframe").toArray()) {
        // Ensure https embeds
        if (HTTPS_ONLY && iframe.attribs["src"]) {
            iframe.attribs["src"] = iframe.attribs["src"].replace(
                "http://",
                "https://"
            )
        }
        // Lazy load unless "loading" attribute already specified
        if (!iframe.attribs["loading"]) {
            iframe.attribs["loading"] = "lazy"
        }
    }

    // Remove any empty elements
    for (const p of cheerioEl("p").toArray()) {
        const $p = cheerioEl(p)
        if ($p.contents().length === 0) $p.remove()
    }

    // Wrap tables so we can do overflow-x: scroll if needed
    for (const table of cheerioEl("table").toArray()) {
        const $table = cheerioEl(table)
        const $div = cheerioEl("<div class='tableContainer'></div>")
        $table.after($div)
        $div.append($table)
    }

    // Due to CSS Grid, we need to nest a container _inside_ the sticky column
    // then put the children of that column inside the container
    function nestStickyContainer(
        $columns: Cheerio,
        side: "left" | "right" = "right"
    ) {
        const parent =
            side === "left"
                ? $columns.children().first()
                : $columns.children().last()
        const container = cheerioEl(`<div class="wp-sticky-container"></div>`)
        container.append(parent.children())
        parent.append(container)
    }

    // Nesting for sticky columns that have been manually created
    ;(["left", "right"] as const).forEach((side) => {
        cheerioEl(`.wp-block-columns.is-style-sticky-${side}`).each(
            (_, columns) => {
                // don't nest the columns when inside related-charts
                const parentClassName = columns.parent.attribs.class
                if (parentClassName === RELATED_CHARTS_CLASS_NAME) {
                    return
                }
                nestStickyContainer(cheerioEl(columns), side)
            }
        )
    })

    // Make sticky-right layout the default for columns
    cheerioEl(".wp-block-columns").each((_, columns) => {
        const $columns = cheerioEl(columns)
        if (columns.attribs.class === "wp-block-columns") {
            $columns.addClass("is-style-sticky-right")
            nestStickyContainer($columns)
        }
    })

    formatImages(cheerioEl)

    // Table of contents and deep links
    const tocHeadings: TocHeading[] = []
    const existingSlugs: string[] = []
    let parentSlug: string | null = null

    cheerioEl("h1, h2, h3, h4").each((_, el) => {
        const $heading = cheerioEl(el)
        const headingText = $heading.text()

        let slug = $heading.attr("id") ?? urlSlug(headingText)

        if (existingSlugs.indexOf(slug) !== -1 && parentSlug) {
            slug = `${parentSlug}-${slug}`
        }

        existingSlugs.push(slug)
        if ($heading.is("h2")) parentSlug = slug

        // Table of contents
        if (formattingOptions.toc) {
            if ($heading.is("#footnotes") && footnotes.length > 0) {
                tocHeadings.push({
                    text: headingText,
                    slug: "footnotes",
                    isSubheading: false,
                })
            } else if ($heading.is("h2")) {
                const tocHeading = {
                    text: headingText,
                    slug: slug,
                    isSubheading: false,
                }
                tocHeadings.push(tocHeading)
            } else if (
                $heading.is("h3") &&
                $heading.closest(`.${PROMINENT_LINK_CLASSNAME}`).length === 0 &&
                $heading.closest(`.${ADDITIONAL_INFORMATION_CLASS_NAME}`)
                    .length === 0
            ) {
                tocHeadings.push({
                    text: headingText,
                    html: $heading.html() || undefined,
                    slug: slug,
                    isSubheading: true,
                })
            }
        }

        if (
            $heading.closest(`.${PROMINENT_LINK_CLASSNAME}`).length || // already wrapped in <a>
            $heading.closest(`.${ADDITIONAL_INFORMATION_CLASS_NAME}`).length || // prioritize clean SSR of AdditionalInformation
            $heading.closest(".wp-block-help").length
        )
            return

        $heading.attr("id", slug)

        $heading.append(`<a class="${DEEP_LINK_CLASS}" href="#${slug}"></a>`)
    })

    return {
        ...post,
        supertitle,
        lastUpdated,
        byline,
        info,
        style,
        footnotes: footnotes,
        tocHeadings: tocHeadings,
        pageDesc: post.excerpt || cheerioEl("p").first().text(),
        html: getBodyHtml(cheerioEl),
    }
}

export const formatPost = async (
    post: FullPost,
    formattingOptions: FormattingOptions,
    knex: KnexReadonlyTransaction
): Promise<FormattedPost> => {
    // No formatting applied, plain source HTML returned
    if (formattingOptions.raw)
        return {
            ...post,
            html: formatUrls(post.content),
            footnotes: [],
            tocHeadings: [],
            pageDesc: post.excerpt || "",
        }

    // Override formattingOptions if specified in the post (as an HTML comment)
    const options: FormattingOptions = {
        toc: post.type === "page",
        footnotes: true,
        ...formattingOptions,
    }

    return formatWordpressPost(post, options, knex)
}