Merge cockroachdb#68836

68836: dev: generate `docs/generated/redact_safe.md` in `dev` r=rail a=rickystewart

This file can't properly be bazelfied because generating it requires
access to the entire source tree, which we can't easily pass into the
Bazel sandbox. Instead it gets special non-Bazel support in `dev` and a
special validation in CI.

Closes cockroachdb#67923

Release note: None

Co-authored-by: Ricky Stewart <[email protected]>

Makefile

@@ -1562,17 +1562,7 @@ bin/.docgen_http: bin/docgen bin/.bootstrap
 .PHONY: docs/generated/redact_safe.md
 
 docs/generated/redact_safe.md:
-	@(echo "The following types are considered always safe for reporting:"; echo; \
-	  echo "File | Type"; echo "--|--") >$@.tmp || { rm -f $@.tmp; exit 1; }
-	@git grep --recurse-submodules -n '^func \(.*\) SafeValue\(\)' | \
-	  grep -v '^vendor/github.com/cockroachdb/redact' | \
-	  sed -E -e 's/^([^:]*):[0-9]+:func \(([^ ]* )?(.*)\) SafeValue.*$$/\1 | \`\3\`/g' | \
-	  sort >>$@.tmp || { rm -f $@.tmp; exit 1; }
-	@git grep --recurse-submodules -n 'redact\.RegisterSafeType' | \
-	  grep -vE '^([^:]*):[0-9]+:[ 	]*//' | \
-	  grep -v '^vendor/github.com/cockroachdb/redact' | \
-	  sed -E -e 's/^([^:]*):[0-9]+:.*redact\.RegisterSafeType\((.*)\).*/\1 | \`\2\`/g' | \
-	  sort  >>$@.tmp || { rm -f $@.tmp; exit 1; }
+	./build/bazelutil/generate_redact_safe.sh >$@.tmp || { rm -f $@.tmp; exit 1; }
 	@mv -f $@.tmp $@
 
 EVENTLOG_PROTOS = \

build/bazelutil/generate_redact_safe.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "The following types are considered always safe for reporting:"
+echo
+echo "File | Type"; echo "--|--"
+git grep --recurse-submodules -n '^func \(.*\) SafeValue\(\)' | \
+    grep -v '^vendor/github.com/cockroachdb/redact' | \
+    sed -E -e 's/^([^:]*):[0-9]+:func \(([^ ]* )?(.*)\) SafeValue.*$$/\1 | \`\3\`/g' | \
+    sort
+git grep --recurse-submodules -n 'redact\.RegisterSafeType' | \
+    grep -vE '^([^:]*):[0-9]+:[ 	]*//' | \
+    grep -v '^vendor/github.com/cockroachdb/redact' | \
+    sed -E -e 's/^([^:]*):[0-9]+:.*redact\.RegisterSafeType\((.*)\).*/\1 | \`\2\`/g' | \
+    sort

build/teamcity/cockroach/ci/builds/build_linux_x86_64.sh

@@ -25,6 +25,7 @@ do
     then
         echo "File $FILE does not match with checked-in version. Got diff:"
         echo "$RESULT"
+        echo "Run './dev generate docs'"
         FAILED=1
     fi
 done
@@ -60,9 +61,17 @@ do
     then
         echo "Generated file $dir/$new_basename does not match with checked-in version. Got diff:"
         echo "$RESULT"
+        echo "Run './dev generate go'"
         FAILED=1
     fi
 done
+# docs/generated/redact_safe.md needs special handling.
+REAL_REDACT_SAFE=$($root/build/bazelutil/generate_redact_safe.sh)
+RESULT=$(diff <(echo "$REAL_REDACT_SAFE") $root/docs/generated/redact_safe.md)
+if [[ ! $? -eq 0 ]]
+then
+    echo "docs/generated/redact_safe.md is not up-to-date. Run './dev generate docs'"
+fi
 
 if [[ ! -z "$FAILED" ]]
 then

pkg/cmd/dev/generate.go

@@ -133,7 +133,12 @@ func (d *dev) generateDocs(cmd *cobra.Command) error {
 			}
 		}
 	}
-	return nil
+	// docs/generated/redact_safe.md needs special handling.
+	output, err := d.exec.CommandContextSilent(ctx, filepath.Join(workspace, "build", "bazelutil", "generate_redact_safe.sh"))
+	if err != nil {
+		return err
+	}
+	return d.os.WriteFile(filepath.Join(workspace, "docs", "generated", "redact_safe.md"), string(output))
 }
 
 func (d *dev) generateGo(cmd *cobra.Command) error {

pkg/cmd/dev/io/os/os.go

@@ -186,6 +186,21 @@ func (o *OS) ReadFile(filename string) (string, error) {
 	return ret, err
 }
 
+// WriteFile wraps around ioutil.ReadFile, writing the given contents to
+// the given file on disk.
+func (o *OS) WriteFile(filename, contents string) error {
+	command := fmt.Sprintf("echo %s > %s", strings.TrimSpace(contents), filename)
+	o.logger.Print(command)
+
+	if o.Recording == nil {
+		// Do the real thing.
+		return ioutil.WriteFile(filename, []byte(contents), 0666)
+	}
+
+	_, err := o.replay(command)
+	return err
+}
+
 // CopyFile copies a file from one location to another.
 func (o *OS) CopyFile(src, dst string) error {
 	command := fmt.Sprintf("cp %s %s", src, dst)

pkg/cmd/dev/testdata/generate.txt

@@ -24,6 +24,8 @@ cp /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/b
 cp /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/docs/generated/sql/functions.md go/src/github.com/cockroachdb/cockroach/docs/generated/sql/functions.md
 cp /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/docs/generated/sql/operators.md go/src/github.com/cockroachdb/cockroach/docs/generated/sql/operators.md
 cp /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/docs/generated/sql/window_functions.md go/src/github.com/cockroachdb/cockroach/docs/generated/sql/window_functions.md
+go/src/github.com/cockroachdb/cockroach/build/bazelutil/generate_redact_safe.sh
+echo MOCK_REDACT_SAFE_OUTPUT > go/src/github.com/cockroachdb/cockroach/docs/generated/redact_safe.md
 
 dev gen go
 ----

pkg/cmd/dev/testdata/recording/generate.txt

@@ -125,6 +125,13 @@ cp /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/b
 cp /private/var/tmp/_bazel/99e666e4e674209ecdb66b46371278df/execroot/cockroach/bazel-out/darwin-fastbuild/bin/docs/generated/sql/window_functions.md go/src/github.com/cockroachdb/cockroach/docs/generated/sql/window_functions.md
 ----
 
+go/src/github.com/cockroachdb/cockroach/build/bazelutil/generate_redact_safe.sh
+----
+MOCK_REDACT_SAFE_OUTPUT
+
+echo MOCK_REDACT_SAFE_OUTPUT > go/src/github.com/cockroachdb/cockroach/docs/generated/redact_safe.md
+----
+
 getenv PATH
 ----
 /usr/local/opt/ccache/libexec:/usr/local/opt/make/libexec/gnubin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin:/Library/Apple/usr/bin