render markdown, strip html tags in indexing (#1148)

* wrap search results

* use templateContent to index algolia content without tags

* fallback to description if needed

* feedback

* fix types

.eleventy.js

@@ -1,5 +1,5 @@
 const yaml = require('js-yaml');
-const {drafts} = require('./site/_utils/drafts');
+const {filterOutDrafts} = require('./site/_utils/drafts');
 
 // Filters
 const {
@@ -41,7 +41,7 @@ const rssPlugin = require('@11ty/eleventy-plugin-rss');
 const syntaxHighlight = require("@11ty/eleventy-plugin-syntaxhighlight");
 
 // Supported locales
-const locales = require('./site/_data/site').locales;
+const locales = require('./site/_data/site.json').locales;
 
 // Collections
 const algoliaCollection = require('./site/_collections/algolia');
@@ -80,7 +80,10 @@ module.exports = eleventyConfig => {
 
   // Add collections
   locales.forEach(locale => eleventyConfig.addCollection(`blog-${locale}`, collections => {
-    let blogCollection = collections.getFilteredByGlob(`./site/${locale}/blog/*/*.md`).filter(drafts).reverse();
+    let blogCollection = collections
+      .getFilteredByGlob(`./site/${locale}/blog/*/*.md`)
+      .filter(filterOutDrafts)
+      .reverse();
     // If we're running inside of Percy then just show the first six blog posts.
     if (process.env.PERCY_BRANCH) {
       blogCollection = blogCollection.slice(blogCollection.length - 6);

algolia.js

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 require('dotenv').config();
-const algoliasearch = require('algoliasearch');
+const {default: algoliasearch} = require('algoliasearch');
 const fs = require('fs');
 const {sizeof} = require('sizeof');
 

cloud-secrets.js

@@ -17,6 +17,9 @@ const cloudSecrets = async () => {
   const [secretsList] = await client.listSecrets({parent: project});
 
   for (const secretItem of secretsList) {
+    if (!secretItem.name) {
+      continue;
+    }
     const key = secretItem.name.split('/').pop();
     const [versions] = await client.listSecretVersions({
       parent: secretItem.name,
@@ -27,7 +30,7 @@ const cloudSecrets = async () => {
       const [accessedSecret] = await client.accessSecretVersion({
         name: version.name,
       });
-      const value = accessedSecret.payload.data.toString();
+      const value = accessedSecret.payload?.data?.toString() ?? '';
       dotenv += `${key}=${value}\n`;
       fetchedSecrets.push(key);
     }

package.json

@@ -64,7 +64,6 @@
     "querystring": "^0.2.0",
     "readdirp": "^3.4.0",
     "redirects-yaml": "^2.0.3",
-    "remove-markdown": "^0.3.0",
     "rollup": "^2.22.2",
     "striptags": "^3.1.1",
     "unistore": "^3.5.2",

site/_collections/algolia.js

@@ -14,12 +14,15 @@
  * limitations under the License.
  */
 
-const removeMarkdown = require('remove-markdown');
-const {createHash} = require('crypto');
+/**
+ * @fileoverview Collection of indexable posts rewritten to our expected format
+ * for indexing via Algolia.
+ */
 
+const {createHash} = require('crypto');
 const {generateImgixSrc} = require('../_shortcodes/Img');
-const {drafts} = require('../_utils/drafts');
 const {stripDefaultLocale} = require('../_filters/urls');
+const striptags = require('striptags');
 
 /**
  * Shrink the size of the given fulltext to fit within a certain limit, at the
@@ -47,38 +50,60 @@ function limitText(content, limit = 7500) {
  * @returns {AlgoliaCollectionItem[]}
  */
 module.exports = collections => {
-  const allSorted = collections.getAllSorted().filter(drafts);
-  /** @type {AlgoliaCollectionItem[]} */
-  const algoliaCollectionItems = [];
+  const toIndex = collections.getAllSorted().filter(item => {
+    const {data} = item;
 
-  for (const item of allSorted) {
-    if (item.data.disable_algolia || item.data.permalink === false) {
-      continue;
+    // Filter out pages we don't want to index. This includes drafts, which we
+    // always filter (even in dev, since we're not really indexing here).
+    if (
+      data.disable_algolia ||
+      data.noindex ||
+      data.draft ||
+      data.permalink === false
+    ) {
+      return false;
     }
 
-    if (item.data.tags && typeof item.data.tags === 'string') {
-      item.data.tags = [item.data.tags];
-    }
+    return true;
+  });
+
+  return toIndex.map(item => {
+    const image = item.data.hero
+      ? generateImgixSrc(item.data.hero, {w: 100, auto: 'format'})
+      : '';
 
     /** @type {AlgoliaCollectionItem} */
     const algoliaCollectionItem = {
       title: item.data.title,
       description: item.data.description,
-      content: limitText(removeMarkdown(item.template.frontMatter.content)),
+      content: undefined,
       url: stripDefaultLocale(item.url),
-      tags: item.data.tags || [],
+      tags: [item.data.tags ?? []].flat(),
       locale: item.data.locale,
-      image:
-        item.data.hero &&
-        generateImgixSrc(item.data.hero, {w: 100, auto: 'format'}),
+      image,
       objectID: createHash('md5').update(item.url).digest('hex'),
     };
 
+    // The item is dumped to JSON, but we can't get at the underlying post's
+    // templateContent until the rendering state of 11ty: the templateContent
+    // prop itself is a getter.
+    // (templateContent returns the final rendered HTML for this page, not
+    // including the layout).
+    // So, define our own getter which will work at that later point.
+    Object.defineProperty(algoliaCollectionItem, 'content', {
+      get() {
+        // Strip HTML tags and limit to a sensible size for indexing.
+        // As stated above, there's no more Markdown in this content.
+        const text = striptags(item.templateContent ?? '');
+        return limitText(text);
+      },
+      enumerable: true,
+    });
+
     if (item.data.type) {
       algoliaCollectionItem.type = item.data.type;
     }
 
-    algoliaCollectionItems.push(algoliaCollectionItem);
-  }
-  return algoliaCollectionItems;
+    return algoliaCollectionItem;
+  });
 };

site/_collections/feeds.js

@@ -16,7 +16,9 @@
 
 const {defaultLocale} = require('../_data/site.json');
 const {i18n} = require('../_filters/i18n');
-const {drafts} = require('../_utils/drafts');
+const {filterOutDrafts} = require('../_utils/drafts');
+
+const MAX_POSTS = 10;
 
 /**
  * @param {EleventyData} data
@@ -43,12 +45,10 @@ function tagsForData(data) {
  * @returns {FeedsCollection} Key value pair of tag name with `FeedsCollectionItem`.
  */
 module.exports = collection => {
-  const MAX_POSTS = 10;
   const posts = collection
     .getAllSorted()
     .reverse()
-    .filter(i => i.data.locale === defaultLocale)
-    .filter(drafts)
+    .filter(i => i.data.locale === defaultLocale && filterOutDrafts(i))
     .filter(item => {
       if (item.data.noindex || item.data.permalink === false) {
         return false;

site/_collections/tags.js

@@ -15,7 +15,7 @@
  */
 const {locales} = require('../_data/site.json');
 const supportedTags = require('../_data/supportedTags.json');
-const {drafts} = require('../_utils/drafts');
+const {filterOutDrafts} = require('../_utils/drafts');
 
 /**
  * Returns an object with the keys being supported tags and the object
@@ -30,7 +30,10 @@ module.exports = function (collections) {
   /** @type Tags */
   const tags = {};
 
-  const allSorted = collections.getAllSorted().reverse().filter(drafts);
+  const allSorted = collections
+    .getAllSorted()
+    .reverse()
+    .filter(filterOutDrafts);
 
   /**
    * Iterates over every post in order to place them in the proper tag collections.

site/_js/web-components/search-box.js

@@ -311,7 +311,12 @@ export class SearchBox extends BaseElement {
             r[highlightKey] = highlightValue.value;
           }
         }
-        r.snippet = r._snippetResult.content.value;
+        // Some pages don't get indexed with fulltext content, but they do have
+        // a meta description, so fall back to that.
+        r.snippet =
+          r._snippetResult.content?.value ||
+          r._snippetResult.description?.value ||
+          '';
         return r;
       });
 

site/_scss/blocks/_search-box.scss

@@ -70,6 +70,8 @@ search-box[active] {
   position: absolute;
   // top = search-box's height + top-nav's bottom padding + gap
   top: calc(#{$search-box-height} + #{nth(map-get($top-nav-padding, 'lg'), 1)} + #{get-size(400)});
+  // Helpful for smaller screens in case we end up with long text in search results.
+  word-break: break-word;
 
   strong {
     color: var(--color-primary);

site/_utils/drafts.js

@@ -1,16 +1,33 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 /**
- * A filter to remove draft pages.
- * We allow drafts to appear in a dev environment, but omit them from
- * collections in a production environment.
+ * A filter to remove draft pages. We allow drafts to appear in a dev
+ * environment, but omit them from collections in a production environment.
+ *
+ * This is purely here as a convenience so drafts still get included in dev.
+ *
  * @param {EleventyCollectionItem} item
  * @return {boolean}
  */
-const drafts = item => {
-  if (process.env.NODE_ENV !== 'production') {
-    return true;
-  } else {
+const filterOutDrafts = item => {
+  if (process.env.NODE_ENV === 'production') {
     return !item.data.draft;
   }
+  return true; // include everything in non-prod
 };
 
-module.exports = {drafts};
+module.exports = {filterOutDrafts};

tsconfig.json

@@ -30,15 +30,16 @@
     "server/**/*",
     "site/**/*",
     "tools/**/*",
-    "types/**/*.d.ts"
+    "types/**/*.d.ts",
+    "*.js",
+    ".eleventy.js",
   ],
   "exclude": [
     ".github",
     ".vscode",
     "dist",
     "node_modules",
     "test",
-    "./*.js",
-    "site/_js/prettify.js"
-  ]
+    "site/_js/prettify.js",
+  ],
 }

types/11ty/collection-item.d.ts

@@ -43,7 +43,7 @@ declare global {
     /**
      * The rendered content of this template. This does not include layout wrappers.
      */
-    templateContent: unknown;
+    templateContent: string;
     /**
      * @UNDOCUMENTED
      */