feat(gatsby): Exclude the internal/fields objects for greater consist…

…ency creating contentDigest (#33671) * feat(gatsby-core-utils): Exclude the internal object for greater consistency I was puzzled why a site was having more nodes than expected by marked as changed on each data update. I use jest-diff to show the diff between the nodes and as it turns out, it was all trivial differences in the `internal` object. Since the `internal` object is managed by the framework and source/transformer plugins typically don't ensure that the `internal` object hasn't changed between updates — we should just exclude it when generating the digest. Making this change resulted in ~20 less nodes being changed which reduced the number of invalidated queries from ~40k to ~6k, dramatically speeding up the incremental builds. I tested whether the destructuring would slow things down any by running the ~700k nodes on this site through both the old and new version of the site. They were essentially the same — ~16.5s vs. ~16.9s with the new algorithm being faster, presumably as we don't now have to hash the internal object. I expect this will result in fewer changed nodes in a lot of plugins. * Update create-content-digest.ts * Only hash as node if node & update snapshots * Only remove autogenerated fields from internal object * Move node-specific createContentDigest code to api-runner-node * revert snapshot changes * update snapshots * Update packages/gatsby/src/utils/api-runner-node.js Co-authored-by: Michal Piechowiak <[email protected]> Co-authored-by: Ward Peeters <[email protected]> Co-authored-by: Michal Piechowiak <[email protected]>
gatsbyjs · Jun 3, 2022 · 7dfd52d · 7dfd52d
1 parent 0cd90d9
commit 7dfd52d
Show file tree

Hide file tree

Showing 8 changed files with 38 additions and 13 deletions.
diff --git a/...s/cache-resilience/plugins/source-and-transformers-child-nodes/source-changed/scenario.js b/...s/cache-resilience/plugins/source-and-transformers-child-nodes/source-changed/scenario.js
@@ -57,8 +57,8 @@ const nodesTest = ({
       -   \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_parentChangeForTransformer\\",
           \\"internal\\": Object {
-      -     \\"contentDigest\\": \\"9d6d458358c77dbe8f4247752ebe41f0\\",
-      +     \\"contentDigest\\": \\"3021b9f76357d1cffb3c40fabc9e08fb\\",
+      -     \\"contentDigest\\": \\"dc3d69faa290879f166b8cf2a459e123\\",
+      +     \\"contentDigest\\": \\"5b2b719234b7823a8a7743409714b7a7\\",
             \\"owner\\": \\"source-and-transformers-child-nodes/source-changed/gatsby-source\\",
             \\"type\\": \\"Parent_ParentChangeForTransformer\\",
           },
@@ -75,8 +75,8 @@ const nodesTest = ({
       +   \\"foo\\": undefined,
           \\"id\\": \\"parent_parentChangeForTransformer >>> Child\\",
           \\"internal\\": Object {
-      -     \\"contentDigest\\": \\"603e50c1fe96279688538ab046d1d70a\\",
-      +     \\"contentDigest\\": \\"f784f9722081b56fee8ca34708299a37\\",
+      -     \\"contentDigest\\": \\"7ec9e8b4f83ebca01990608f961b1a5d\\",
+      +     \\"contentDigest\\": \\"4cecf7ebaf0c19b22eef8835c725c2e0\\",
             \\"owner\\": \\"source-and-transformers-child-nodes/source-changed/gatsby-transformer\\",
             \\"type\\": \\"ChildOfParent_ParentChangeForTransformer\\",
           },

diff --git a/...ache-resilience/plugins/source-and-transformers-child-nodes/transformer-added/scenario.js b/...ache-resilience/plugins/source-and-transformers-child-nodes/transformer-added/scenario.js
@@ -55,7 +55,7 @@ const nodesTest = ({
           \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_childAdditionForTransformer\\",
           \\"internal\\": Object {
-            \\"contentDigest\\": \\"f85e860f002547e9da9e893e3e44e162\\",
+            \\"contentDigest\\": \\"f99c0539a6bb6061b072fe782a8e441f\\",
             \\"owner\\": \\"source-and-transformers-child-nodes/transformer-added/gatsby-source\\",
             \\"type\\": \\"Parent_ChildAdditionForTransformer\\",
           },

diff --git a/...he-resilience/plugins/source-and-transformers-child-nodes/transformer-changed/scenario.js b/...he-resilience/plugins/source-and-transformers-child-nodes/transformer-changed/scenario.js
@@ -78,8 +78,8 @@ const nodesTest = ({
       +   \\"foo\\": \\"baz\\",
           \\"id\\": \\"parent_childChangeForTransformer >>> Child\\",
           \\"internal\\": Object {
-      -     \\"contentDigest\\": \\"25ad44d8db6f7cee17e248d3b4c94538\\",
-      +     \\"contentDigest\\": \\"3c57fc662e03b2b443b47575f7e82ec0\\",
+      -     \\"contentDigest\\": \\"04b03439f5e1d249d999e6763e23f306\\",
+      +     \\"contentDigest\\": \\"66875f4e1170801ce795d9d863a12b2a\\",
             \\"owner\\": \\"source-and-transformers-child-nodes/transformer-changed/gatsby-transformer\\",
             \\"type\\": \\"ChildOfParent_ChildChangeForTransformer\\",
           },

diff --git a/...he-resilience/plugins/source-and-transformers-child-nodes/transformer-removed/scenario.js b/...he-resilience/plugins/source-and-transformers-child-nodes/transformer-removed/scenario.js
@@ -85,7 +85,7 @@ const nodesTest = ({
           \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_childDeletionForTransformer\\",
           \\"internal\\": Object {
-            \\"contentDigest\\": \\"872081fdfb66891ee6ccdcd13716a5ce\\",
+            \\"contentDigest\\": \\"6140f27e9454bda710ae6da2cda698f0\\",
             \\"owner\\": \\"source-and-transformers-child-nodes/transformer-removed/gatsby-source\\",
             \\"type\\": \\"Parent_ChildDeletionForTransformer\\",
           },

diff --git a/...s/cache-resilience/plugins/source-and-transformers-node-fields/source-changed/scenario.js b/...s/cache-resilience/plugins/source-and-transformers-node-fields/source-changed/scenario.js
@@ -49,8 +49,8 @@ const nodesTest = ({
       -   \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_parentChangeForFields\\",
           \\"internal\\": Object {
-      -     \\"contentDigest\\": \\"e88540d53597617cf99d612601037013\\",
-      +     \\"contentDigest\\": \\"3b78e62e87d3f1d8e92d274aa8dbe548\\",
+      -     \\"contentDigest\\": \\"bde8140cd815ea2bb9e7ae2b45330d08\\",
+      +     \\"contentDigest\\": \\"76dc03ca64a18a99d1581b9e7dd224ef\\",
             \\"fieldOwners\\": Object {
               \\"bar\\": \\"source-and-transformers-node-fields/source-changed/gatsby-transformer\\",
               \\"foo\\": \\"source-and-transformers-node-fields/source-changed/gatsby-transformer\\",

diff --git a/...ache-resilience/plugins/source-and-transformers-node-fields/transformer-added/scenario.js b/...ache-resilience/plugins/source-and-transformers-node-fields/transformer-added/scenario.js
@@ -50,7 +50,7 @@ const nodesTest = ({
           \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_childAdditionForFields\\",
           \\"internal\\": Object {
-            \\"contentDigest\\": \\"bdf44fdce30b104b4f290d66c2dc3ca1\\",
+            \\"contentDigest\\": \\"b6d6df9ccdfc5388cde03e3140520cf5\\",
       +     \\"fieldOwners\\": Object {
       +       \\"foo1\\": \\"source-and-transformers-node-fields/transformer-added/gatsby-transformer\\",
       +     },

diff --git a/...he-resilience/plugins/source-and-transformers-node-fields/transformer-changed/scenario.js b/...he-resilience/plugins/source-and-transformers-node-fields/transformer-changed/scenario.js
@@ -67,7 +67,7 @@ const nodesTest = ({
           \\"foo\\": \\"run-1\\",
           \\"id\\": \\"parent_childChangeForFields\\",
           \\"internal\\": Object {
-            \\"contentDigest\\": \\"fb9e9b9c26522bceaa1f51c537b2aff2\\",
+            \\"contentDigest\\": \\"c6c5c686a949351323ba4a04b585840a\\",
             \\"fieldOwners\\": Object {
       -       \\"foo1\\": \\"source-and-transformers-node-fields/transformer-changed/gatsby-transformer\\",
       +       \\"foo2\\": \\"source-and-transformers-node-fields/transformer-changed/gatsby-transformer\\",

diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js
@@ -13,7 +13,9 @@ const { codeFrameColumns } = require(`@babel/code-frame`)
 const fs = require(`fs-extra`)
 const { getCache } = require(`./get-cache`)
 import { createNodeId } from "./create-node-id"
-const { createContentDigest } = require(`gatsby-core-utils`)
+const {
+  createContentDigest: _createContentDigest,
+} = require(`gatsby-core-utils`)
 import {
   buildObjectType,
   buildUnionType,
@@ -32,6 +34,29 @@ const { trackBuildError, decorateEvent } = require(`gatsby-telemetry`)
 import errorParser from "./api-runner-error-parser"
 import { wrapNode, wrapNodes } from "./detect-node-mutations"
 
+// Override createContentDigest to remove autogenerated data from nodes to
+// ensure consistent digests.
+function createContentDigest(node) {
+  if (!node?.internal?.type) {
+    // this doesn't look like a node, so let's just pass it as-is
+    return _createContentDigest(node)
+  }
+  return _createContentDigest({
+    ...node,
+    internal: {
+      ...node.internal,
+      // Remove auto-generated fields that'd prevent
+      // creating a consistent contentDigest.
+      contentDigest: undefined,
+      owner: undefined,
+      fieldOwners: undefined,
+      ignoreType: undefined,
+      counter: undefined,
+    },
+    fields: undefined,
+  })
+}
+
 if (!process.env.BLUEBIRD_DEBUG && !process.env.BLUEBIRD_LONG_STACK_TRACES) {
   // Unless specified - disable longStackTraces
   // as this have severe perf penalty ( http://bluebirdjs.com/docs/api/promise.longstacktraces.html )