Use robots as cue to remove algolia index

zxc00o · Aug 12, 2020 · b0c2c77 · b0c2c77
1 parent 7bf73d3
commit b0c2c77
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -65,8 +65,19 @@ support_level: production
 This is my  content.
 ```
 
+## Search
+
+Search is powered by Algolia, and will index all content in /docs/ that is Markdown or MDX formatted.
+
+It will _not_ index documents with any of the following in their frontmatter:
+
+- `draft: true`
+- `noindex: true`
+- `robots: noindex`
+
 ## Notes on Markdown vs MDX
 
+
 :pray: that MDX v2 fixes this.
 
 MDX has its flaws. When rendering components, any text inside of them is treated as raw text (_not_ markdown). To work around this you can use the `<markdown>` tag, but it also has its issues. Generally speaking, put an empty line after the opening tag, and before the closing tag.

diff --git a/gatsby-node.js b/gatsby-node.js
@@ -69,9 +69,13 @@ exports.createSchemaCustomization = ({ actions, schema }) => {
         redirect_from: {
           type: "[String!]",
         },
+        // TODO: we can probably combine noindex + robots
         noindex: {
           type: "Boolean",
         },
+        robots: {
+          type: "String",
+        },
         sidebar_order: {
           type: "Int",
           resolve(source, args, context, info) {

diff --git a/src/utils/algolia.js b/src/utils/algolia.js
@@ -12,6 +12,7 @@ const pageQuery = `{
               title
               draft
               noindex
+              robots
             }
             fields {
               slug
@@ -24,6 +25,7 @@ const pageQuery = `{
               title
               draft
               noindex
+              robots
             }
             fields {
               slug
@@ -42,15 +44,19 @@ const flatten = arr =>
       ({ node: { childMarkdownRemark, childMdx } }) =>
         childMarkdownRemark || childMdx
     )
-    .map(
-      ({ node: { childMarkdownRemark, childMdx, objectID } }) => (
-        childMarkdownRemark || childMdx, objectID
-      )
+    .map(({ node: { childMarkdownRemark, childMdx, objectID } }) => [
+      childMarkdownRemark || childMdx,
+      objectID,
+    ])
+    .filter(
+      ([child, _]) =>
+        !child.frontmatter.noindex &&
+        !child.frontmatter.draft &&
+        child.frontmatter.robots !== "noindex"
     )
-    .filter(([child]) => !child.frontmatter.noindex)
     .map(([child, objectID]) => ({
       objectID,
-      ...child.frontmatter,
+      title: child.frontmatter.title,
       url: child.fields.slug,
       content: child.excerpt,
       score: child.legacy ? 0 : 1,