From b1cad36d680d64ce1d70e69da801495c64565b0e Mon Sep 17 00:00:00 2001 From: Sam Nixon Date: Thu, 19 Sep 2024 13:39:19 +0200 Subject: [PATCH 1/9] Add new sitemap for images to resolve indexing issues with webp images --- gatsby-config.js | 62 +++++++++++++++--------------------------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/gatsby-config.js b/gatsby-config.js index cceb1428d..aefd024c4 100644 --- a/gatsby-config.js +++ b/gatsby-config.js @@ -57,28 +57,6 @@ const getContentfulOptions = () => { }; }; -// These paths are blocked from search engine indexing and search engine access. -const DISLALLOW_LIST = [ - '/purchase/', - '/purchase/success/', - '/tailwind/404/', - '/installation-pending/', - - // These are partials which eventually get embedded into other pages. - '/docs/details/create-proxy/structure/header/', - '/docs/getting-started/adding-a-catalog-item/structure/header/', - '/docs/details/create-proxy/basic/', - - // These are partials which eventually get embedded into other pages. - '/docs/getting-started/adding-a-catalog-item/bitbucket/', - '/docs/getting-started/adding-a-catalog-item/github/', - '/docs/getting-started/adding-a-catalog-item/gitlab/', - '/docs/getting-started/adding-a-catalog-item/roadie-api/', - '/docs/getting-started/adding-a-catalog-item/aws-s3/', - '/docs/getting-started/adding-a-catalog-item/azure-devops/', - '/docs/getting-started/adding-a-catalog-item/roadie-cli/', -]; - // Only environment variables prefixed with GATSBY_ are available in the runtime. Here we turn // a server side variable into a runtime one. This variable is later used to determine which // branch of a split testing experiment we are on. @@ -111,7 +89,24 @@ module.exports = { ignore: ['**/.*', '**/template.md'], }, }, - + { + resolve: `gatsby-plugin-sitemap`, + options: { + output: `/sitemap-images`, + serialize: ({ path, allFile }) => { + const images = allFile.nodes.map((node) => ({ + url: `https://roadie.io${node.publicURL}`, + })); + + return { + url: path, + changefreq: `daily`, + priority: 0.7, + images, + }; + }, + }, + }, { resolve: `gatsby-plugin-algolia`, options: { @@ -206,26 +201,7 @@ module.exports = { }, }, - { - resolve: 'gatsby-plugin-sitemap', - options: { - excludes: DISLALLOW_LIST, - }, - }, - - { - resolve: 'gatsby-plugin-robots-txt', - options: { - host: getSiteUrl(), - sitemap: `${getSiteUrl()}/sitemap-index.xml`, - policy: [ - { - userAgent: '*', - disallow: DISLALLOW_LIST, - }, - ], - }, - }, + 'gatsby-plugin-advanced-sitemap', { resolve: 'gatsby-plugin-react-svg', From 56256a4597ed2fdcde839a17f665905030fb4efd Mon Sep 17 00:00:00 2001 From: Sam Nixon Date: Thu, 19 Sep 2024 13:48:46 +0200 Subject: [PATCH 2/9] Merge latest changes from main --- gatsby-config.js | 87 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 18 deletions(-) diff --git a/gatsby-config.js b/gatsby-config.js index aefd024c4..a0b9257ba 100644 --- a/gatsby-config.js +++ b/gatsby-config.js @@ -57,6 +57,28 @@ const getContentfulOptions = () => { }; }; +// These paths are blocked from search engine indexing and search engine access. +const DISLALLOW_LIST = [ + '/purchase/', + '/purchase/success/', + '/tailwind/404/', + '/installation-pending/', + + // These are partials which eventually get embedded into other pages. + '/docs/details/create-proxy/structure/header/', + '/docs/getting-started/adding-a-catalog-item/structure/header/', + '/docs/details/create-proxy/basic/', + + // These are partials which eventually get embedded into other pages. + '/docs/getting-started/adding-a-catalog-item/bitbucket/', + '/docs/getting-started/adding-a-catalog-item/github/', + '/docs/getting-started/adding-a-catalog-item/gitlab/', + '/docs/getting-started/adding-a-catalog-item/roadie-api/', + '/docs/getting-started/adding-a-catalog-item/aws-s3/', + '/docs/getting-started/adding-a-catalog-item/azure-devops/', + '/docs/getting-started/adding-a-catalog-item/roadie-cli/', +]; + // Only environment variables prefixed with GATSBY_ are available in the runtime. Here we turn // a server side variable into a runtime one. This variable is later used to determine which // branch of a split testing experiment we are on. @@ -89,24 +111,7 @@ module.exports = { ignore: ['**/.*', '**/template.md'], }, }, - { - resolve: `gatsby-plugin-sitemap`, - options: { - output: `/sitemap-images`, - serialize: ({ path, allFile }) => { - const images = allFile.nodes.map((node) => ({ - url: `https://roadie.io${node.publicURL}`, - })); - return { - url: path, - changefreq: `daily`, - priority: 0.7, - images, - }; - }, - }, - }, { resolve: `gatsby-plugin-algolia`, options: { @@ -201,7 +206,53 @@ module.exports = { }, }, - 'gatsby-plugin-advanced-sitemap', + { + resolve: 'gatsby-plugin-sitemap', + options: { + output: `/sitemap-index`, + excludes: DISLALLOW_LIST, + query: ` + { + allSitePage { + nodes { + path + } + } + allFile(filter: { extension: { regex: "/(jpg|jpeg|png|webp)/" } }) { + nodes { + publicURL + } + } + } + `, + serialize: ({ path, allFile }) => { + const images = allFile.nodes.map((node) => ({ + url: `https://roadie.io${node.publicURL}`, + })); + + return { + url: path, + changefreq: `daily`, + priority: 0.7, + images, + }; + }, + }, + }, + + { + resolve: 'gatsby-plugin-robots-txt', + options: { + host: getSiteUrl(), + sitemap: `${getSiteUrl()}/sitemap-index.xml`, + policy: [ + { + userAgent: '*', + disallow: DISLALLOW_LIST, + }, + ], + }, + }, { resolve: 'gatsby-plugin-react-svg', From a6078f98bf014d587161e5aa2fbeed789db4f643 Mon Sep 17 00:00:00 2001 From: Sam Nixon Date: Thu, 19 Sep 2024 19:00:44 +0200 Subject: [PATCH 3/9] Fix build error --- gatsby-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gatsby-config.js b/gatsby-config.js index a0b9257ba..819d58691 100644 --- a/gatsby-config.js +++ b/gatsby-config.js @@ -209,7 +209,7 @@ module.exports = { { resolve: 'gatsby-plugin-sitemap', options: { - output: `/sitemap-index`, + output: `/sitemap`, excludes: DISLALLOW_LIST, query: ` { From b0264f08fbf91312e3524254a2d5d61d8c01ff30 Mon Sep 17 00:00:00 2001 From: Sam Nixon Date: Thu, 19 Sep 2024 19:09:40 +0200 Subject: [PATCH 4/9] Revert and make sure it actually builds --- gatsby-config.js | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/gatsby-config.js b/gatsby-config.js index 819d58691..cceb1428d 100644 --- a/gatsby-config.js +++ b/gatsby-config.js @@ -209,34 +209,7 @@ module.exports = { { resolve: 'gatsby-plugin-sitemap', options: { - output: `/sitemap`, excludes: DISLALLOW_LIST, - query: ` - { - allSitePage { - nodes { - path - } - } - allFile(filter: { extension: { regex: "/(jpg|jpeg|png|webp)/" } }) { - nodes { - publicURL - } - } - } - `, - serialize: ({ path, allFile }) => { - const images = allFile.nodes.map((node) => ({ - url: `https://roadie.io${node.publicURL}`, - })); - - return { - url: path, - changefreq: `daily`, - priority: 0.7, - images, - }; - }, }, }, From 407910dbf4dec37f913a21b9555c91977c269d7c Mon Sep 17 00:00:00 2001 From: Sam Nixon Date: Fri, 20 Sep 2024 16:07:25 +0200 Subject: [PATCH 5/9] add gatsby-plugin-image ref in the gatsby-config --- gatsby-config.js | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/gatsby-config.js b/gatsby-config.js index cceb1428d..ac2af6f91 100644 --- a/gatsby-config.js +++ b/gatsby-config.js @@ -4,6 +4,7 @@ const agoliaQueries = require('./src/queries/agolia'); const rssFeedPlugin = require('./src/gatsby/rssFeedPlugin'); const theme = require('./src/theme'); const GATSBY_PLUGIN_CSP_DIRECTIVES = require('./src/gatsby/cspDirectives'); +const { getSrc } = require('gatsby-plugin-image'); const SITE_TITLE = 'Roadie'; @@ -209,7 +210,49 @@ module.exports = { { resolve: 'gatsby-plugin-sitemap', options: { + output: '/sitemap.xml', excludes: DISLALLOW_LIST, + query: ` + { + allSitePage { + nodes { + path + } + } + allFile(filter: { extension: { regex: "/(jpg|jpeg|png|gif)/" } }) { + nodes { + publicURL + name + childImageSharp { + gatsbyImageData + } + } + } + } + `, + resolvePages: ({ allSitePage: { nodes: allPages }, allFile: { nodes: allImages } }) => { + return allPages.map((page) => { + const images = allImages + .filter((image) => page.path.includes(image.name)) + .map((image) => ({ + url: getSrc(image.childImageSharp.gatsbyImageData), + title: image.name, + })); + + return { ...page, images }; + }); + }, + serialize: ({ path, images }) => { + return { + url: path, + changefreq: 'daily', + priority: 0.7, + images: images.map((image) => ({ + url: image.url, + title: image.title, + })), + }; + }, }, }, From 4b63c32e5b1d8352b45f6f3954897be1c4e0864e Mon Sep 17 00:00:00 2001 From: Sam Nixon Date: Fri, 20 Sep 2024 16:27:03 +0200 Subject: [PATCH 6/9] Try and resolve build error where sitemap doesn't have access to the site url --- gatsby-config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/gatsby-config.js b/gatsby-config.js index ac2af6f91..2a72a1ba1 100644 --- a/gatsby-config.js +++ b/gatsby-config.js @@ -212,6 +212,7 @@ module.exports = { options: { output: '/sitemap.xml', excludes: DISLALLOW_LIST, + resolveSiteUrl: () => getSiteUrl(), query: ` { allSitePage { From 49fb46644bacb87dd57a021689f9f3cd56a375cc Mon Sep 17 00:00:00 2001 From: Sam Nixon Date: Fri, 20 Sep 2024 17:01:52 +0200 Subject: [PATCH 7/9] Try and handle images without data --- gatsby-config.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gatsby-config.js b/gatsby-config.js index 2a72a1ba1..d2fe671ba 100644 --- a/gatsby-config.js +++ b/gatsby-config.js @@ -236,9 +236,13 @@ module.exports = { const images = allImages .filter((image) => page.path.includes(image.name)) .map((image) => ({ - url: getSrc(image.childImageSharp.gatsbyImageData), + url: image.publicURL, // Use publicURL as a fallback title: image.name, - })); + gatsbyImageData: image.childImageSharp + ? getSrc(image.childImageSharp.gatsbyImageData) + : null, // Check for gatsbyImageData + })) + .filter((image) => image.gatsbyImageData); // Filter out images without gatsbyImageData return { ...page, images }; }); From 6dfe68746193c65c21c91afe2e21c37bf74b4e18 Mon Sep 17 00:00:00 2001 From: Sam Nixon Date: Fri, 20 Sep 2024 17:08:25 +0200 Subject: [PATCH 8/9] Fix var name so it's not DISLallow --- gatsby-config.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gatsby-config.js b/gatsby-config.js index d2fe671ba..8627759db 100644 --- a/gatsby-config.js +++ b/gatsby-config.js @@ -59,7 +59,7 @@ const getContentfulOptions = () => { }; // These paths are blocked from search engine indexing and search engine access. -const DISLALLOW_LIST = [ +const DISALLOW_LIST = [ '/purchase/', '/purchase/success/', '/tailwind/404/', @@ -211,7 +211,7 @@ module.exports = { resolve: 'gatsby-plugin-sitemap', options: { output: '/sitemap.xml', - excludes: DISLALLOW_LIST, + excludes: DISALLOW_LIST, resolveSiteUrl: () => getSiteUrl(), query: ` { @@ -269,7 +269,7 @@ module.exports = { policy: [ { userAgent: '*', - disallow: DISLALLOW_LIST, + disallow: DISALLOW_LIST, }, ], }, From 1b7af377cf5efdda6619910565f9e23ec34f65c0 Mon Sep 17 00:00:00 2001 From: Sam Nixon Date: Tue, 1 Oct 2024 22:04:18 +0200 Subject: [PATCH 9/9] Tweak to better handle errors --- gatsby-config.js | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/gatsby-config.js b/gatsby-config.js index 8627759db..b7d390a5b 100644 --- a/gatsby-config.js +++ b/gatsby-config.js @@ -210,7 +210,7 @@ module.exports = { { resolve: 'gatsby-plugin-sitemap', options: { - output: '/sitemap.xml', + output: '/sitemap', excludes: DISALLOW_LIST, resolveSiteUrl: () => getSiteUrl(), query: ` @@ -220,7 +220,7 @@ module.exports = { path } } - allFile(filter: { extension: { regex: "/(jpg|jpeg|png|gif)/" } }) { + allFile(filter: { extension: { regex: "/(jpg|jpeg|png|gif|webp)/" } }) { nodes { publicURL name @@ -235,16 +235,25 @@ module.exports = { return allPages.map((page) => { const images = allImages .filter((image) => page.path.includes(image.name)) - .map((image) => ({ - url: image.publicURL, // Use publicURL as a fallback - title: image.name, - gatsbyImageData: image.childImageSharp - ? getSrc(image.childImageSharp.gatsbyImageData) - : null, // Check for gatsbyImageData - })) - .filter((image) => image.gatsbyImageData); // Filter out images without gatsbyImageData - - return { ...page, images }; + .map((image) => { + // Use publicURL for unsupported formats like GIFs + if (image.extension === 'gif') { + return { + url: image.publicURL || '', // Ensure publicURL fallback + title: image.name, + }; + } + // Use gatsbyImageData for supported formats + return { + url: image.childImageSharp + ? getSrc(image.childImageSharp.gatsbyImageData) + : image.publicURL || '', // Ensure publicURL fallback + title: image.name, + }; + }) + .filter((image) => image.url); // Filter out images without valid URLs + + return { ...page, images: images.length ? images : [] }; }); }, serialize: ({ path, images }) => {