Merge #65874

65874: opt: add exploration steps to optstepsweb r=RaduBerinde a=RaduBerinde

Adding visual `exploretrace` output to `optstepsweb`. The page now
shows a selector that can be toggled between normalization and
exploration.

Release note: None

Co-authored-by: Radu Berinde <[email protected]>

pkg/sql/opt/norm/testdata/rules/combo

@@ -536,7 +536,7 @@ Final best expression
 optstepsweb
 SELECT s FROM a INNER JOIN xy ON a.k=xy.x AND i+1=10
 ----
-https://raduberinde.github.io/optsteps.html?eJzsWl9v4kYQf--nmPJkCztg4I7gylLaNJFyoiRN2qfjFDlgmk3sNbJNC1f1dLrPwGM_XT5JZWOMYXex1_9CmuQpeNazOzs7v9_MeP-u3fzar6m1m7P-2elv4ML59eUvoMPFYHB2DR8uLwYwX8DlAPSjR22-OJrDj4OfAdUVTWnWpNrAdqwxmkxqak2WZdAbFxh5SDeHuF6vw93m58kJyArUFUnpwcnJENenjv1gjLwhrsPT8t-n5den5VcY2ebMwq4KrtrZliDLmnn6nWmsHi_XjzE2HPnBRhiEkWO7rujLgz-K1kdV-R7PTBOQ2oKJ2vangQf1HehHI2d8d4uwZzhYN2-tP0ejWw9Zhuvp1lR9D3O1u3pzoR7DfLF_eI-yhq3178gejYUKgiJ1aYufjH2ZKMvDWbPZNoSW_F6UQGhLHfHLl_CZIrWkd1LwvLsZeSz1aArdkY5Bjwm-Fb1bbN2hpSI5YpnBXJZx80UK6zJ7lGpSN8kktl-iYcj0DMeNJDti4VFVQPOXLQYhKAi-W-qgiKCB0hThoz3zDEcLdqcrSpsTJ8HIxq7n6Ah7riY0FBWExuD3fh9k-PQDNLrx3-KnIV5Fsh_auok-G6fW9Mqcuae-knVcM4RBlPshvgl0iAKd6oYg0BmBAgmBLh9ioFPXsPqn1IXAHsQhZHHEIYR5EYdQuEYcKBFxWLo3iLM7Ihvi0I3zESfRusweZSLOXpPYfiEQR35uxEkJef78GghKE2RQxBwLaKWDwHPbHP-EsO4s1rgXf7IFdt3UYCdnAzvi4D0X0BComxZoiBcLT20ChTF9_lsx_W2q_jcYe2UwVh2KcMFaj1xJ4vQfGz2QodEjZ5dgMta2YiFCtRtkTU00WXywET6Pks4A3qiiEOe6nEmdnFim0eX3unt_0HiXjGlFQRABcNbM9NDURCPkLVQwjYknO_ZfrvDZcGzZdmQbG6IEDvrjniJgVp7surMYQ4I3NU3ormSatg1kb_j6f8TXCuBsG14ju9fLic2fPDttOuLQMiE9NDmasUXMt22d50foPoy-mrn3KwC-wJ7tw3HfmETVN0saz0hbHa7y-yUgMbGQnHBIHh0yZCqDQ4JJ1nAolwiHZP_MMMOmLMek_K5PA8S7dufsEH4ryJ9slazGagX91dRt1kL3ktW5DAeUh5GvgF2rYTM6oxfoKTpdlmzUikGdGTbW1Hhqm1GNQxGseLPVCXjzuCjeTNeeroA3eXrU_iIOmJfLrLfS8nJIkVy8nIUieUuyTmqbCM2xj7HcRoWhwEvnBRAxSyUrcyqBiBOmYJufi4iZqdEOl5ROxBmzq86eZIKegZaDpAmzsNKooiAoQS89n-SIzyL2ImVmyTiPOdJLYBxtyqjyzjfdJy880aTvWmU52U0Q4ERGFn8c5mPHxeZjPChy4GkQLwalTw0IzbF954AertSANSIFi7Bigp6kpfd_gSzCtz9Ul5MJBJ_7M1MQtwdocElkYhznqAQKo8_DdkXKlI5-DgtgwCSNEtMLLOR_484XyJ1nJrIQ1j3jKqrVAvIkn2-x57N8BXi17Ek0C8jCereNwAVTVdEqq8DMS6vhxYA9LxdDeKQNNCzKRUxsU4olDIlpS5qqf2doBaV_rq8g4bX_vW9m_7gh7es7vH3EeNn8GLX1r_0tpzb8Y5Ktjn_7OTr-WTgyayP_oFv0OdrWaTSnunl0yFzMepV1XSprucOx-gwlSbpqpGKgTfyEk_muIBNo995qYF8ATFogm9coGH8QqF3757v_AgAA__8LZehF
+https://raduberinde.github.io/optsteps.html?eJzsnN1u2zYYhs93Fd98JMFSEzlplWgQkC1NhxSp2yUbdpAEherQDRtZMiR5szusKHoNOdzV5UoGybItmaRF6i9yox0tIkXy48_zvqSp_tO5-O2sY3QuTs5Ojn8HH16dv30DFpz2-yfn8PrtaR-mM3jbB-vZnTmdPZvCz_2XgLuaqe12lE7f9UYv8XDYMTqqqoK1c-rgAFv2ldPtduHD6s-jI1A16GqKdghHR1dOd-y5n9AguHK68HD_38P914f7rzBw7cnI8Q3wjf10Ch6NJoH1wUbzx_eLx46DPPWTix2QBp7r-3KYHv1HKfXO0H50JrYN2OjB0NgLq4FPxnOwng28mw_vsRMgz7Hs96O_BoP3AR4hP7BGY-MFTA19_ubMOIDpbHP2Q0obUu1fS7tDMwMkTdFpjR_ehGmyql5Ndnf3kNRTX8gKSHvKvvzlS_xMU3rKcyV6rq9yHiiHtAL9geWAlUj4VnZvscuOI5XJHPc5wmUFN51xRJd7RKkh6VkhscdlmQ3bAfL8ZcpasnRnaGCGzZajJShJ4bB0QZPBBG1Xhkt3EiDPjHpHl5XVjFNg4Dp-4FnYCXxT2tEMkHb6f5ydgQrXP8GOnvxbvr5y5is5XNqWjT-j49H4nT3xj8NCFuuakRit8nCJrxY6LBc6dRiihc5YKJCx0NUmLnRqG-b_U2lDYANxiLQkcYjEosQhClwQByokDqvsFXHWc-QjDj24kDiZ0eUeUSZxNobEHheCOOpjE4cTeWH9JkjaLqigyQUa0OND4CvXvvkFO5Y3W3Av-SQFO50bdmo-2BET77FAQ1CXFzTEi6Vbm6jARHnhW4ny96jltxh7YhirjyJCWDskW5JZ_eXOIaiwc0jWrsDwxkythSXVLvBobOPh7LWLnVdL0xnhjZoUc04XNHVq5jaNnn5r-beN5l0208pCEAG40cQO8NjGAxzMDLDRMFA9929f-ow8V3U91XWQrICHP95SEpg7T_a-s5xAojdNU9LnaaaZBlnL1--RrzXgLI3XZdyL5iTqz66dVh0xaZlIj0Ne1tgj6ktHF4QrdBOj30382zmAT53ADXF8hobL3TcrNelIe_tC2-9tIDHRkII4JKcOuWRqwyGhJAscqhXikDw_Q3Z8KCtQqfjQ84B4Pe6CJ4TfShpPdpGsg9Uazle5j1lL7UvWyWWcoTpGPgF1rUfN6Ipe4kjR5bLioOYK6k0ctJDGY9de7nEoCXPd7O1HunlQlm7yHU_XoJsiZ9RhIxqsy1Xut3h1OZZIIV3OI5GiW7J97piIkhM_xgoHFS8FUTkvQYhZRbKcUwVCnFEFO_xCQsy0RmtaUrkQ53RX-xvMBN2BVkPSjFpYNqosBGWUS_eTAuuzjL7gdJaM-VjAXgJjalNyVTe_6WOy5UaT3mu1ebKLaIETjiz5OPZjB-X6MRGKNNwGiTKI3xoQJSf6XQA9QtaAlYNDRVhrgm7S-Me_RBUR6x_qkJMGQmz4c0uQ8AjQcEk4MYF5VIGE0ethDwWnpaPPwxIUMKtEhTkKLPK32rmF2nli4xF2rAC9W-7VIvEkn6fU81F-BXiy6kkcFpAb6_VjBCFM1SWrrA1mUVmNLwZseLkcwSNjoLGokDCxQylXMBRmLDy7_rWsNWz9C_0KEl_73_hm_h83lE3nDu2PGNutj8tj_fOwy6kH_omU1In_3mOc-OfRyLwH-Y0-oi9wbM1TMtfNoyZrMetV1nWpvNsdgdbn2JLw7UZqBm3mTzi57woyQbvxVgP7AmBWA9m6RmF8I6jdUTon07Hteij1_eA5cr0b5IW4XgI8_SxG91656CYmSRPRSiTlBGAJn80JTbiWs0-Bs_QbbEKcZfOxqRibU-tX5CDPCtAb5H1EKXbRUlIEe1H4Qn1MMNZ9-lFYcyHCEQNb9jX0sASIGI-djwZ0NUqeqLRkJr0WVG4Xw1j4TvYtuONA6slwaQ2CiWWHj64bQT_RKxLU7Tytz3hfuqf0VoP4Sk5F4QMDHr6qTL7SbwvWct9-g6Nm-ZA0Klif24MUeJMQSmmQn7nu3WRMJXkqKYXy3apRbkdVw3TWbJzfodmqNZfaNZhwqV_TuD-PJ84LlofCd1uut950S71pQ9nJx7zWvLbmtTWvrXltzWtrXlvz2prX1rw2hJ2teW3Na03mVecxr9ojQa4Sx1CWRm8Xk1uv_R177a0GecKh_omD-J852WBjk5mSeNf4bz4XM7RWswmf9rN65Ge1xvvZHKhnLb8NN9MKfMpNuf7LgyoBRFfydTO12eWCtjEfJxNlNwGD1YXb-feH_wMAAP__s73s4Q==
 
 # Exploration patterns with varying costs.
 optsteps

pkg/sql/opt/testutils/opttester/opt_tester.go

@@ -1328,11 +1328,26 @@ func (ot *OptTester) OptSteps() (string, error) {
 // OptStepsWeb is similar to Optsteps but it uses a special web page for
 // formatting the output. The result will be an URL which contains the encoded
 // data.
-//
-// TODO(radu): currently, we only show normalization steps on this page.
 func (ot *OptTester) OptStepsWeb() (string, error) {
-	ot.builder.Reset()
+	normDiffStr, err := ot.optStepsNormDiff()
+	if err != nil {
+		return "", err
+	}
+
+	exploreDiffStr, err := ot.optStepsExploreDiff()
+	if err != nil {
+		return "", err
+	}
+	url, err := ot.encodeOptstepsURL(normDiffStr, exploreDiffStr)
+	if err != nil {
+		return "", err
+	}
+	return url.String(), nil
+}
 
+// optStepsNormDiff produces the normalization steps as a diff where each step
+// is a pair of "files" (showing the before and after plans).
+func (ot *OptTester) optStepsNormDiff() (string, error) {
 	// Store all the normalization steps.
 	type step struct {
 		Name string
@@ -1357,9 +1372,7 @@ func (ot *OptTester) OptStepsWeb() (string, error) {
 		normSteps = append(normSteps, step{Name: name, Expr: expr})
 	}
 
-	// Produce a diff for each step, as if there was a pair of files for each step
-	// (showing the before and after plans).
-	var normDiff bytes.Buffer
+	var buf bytes.Buffer
 	for i, s := range normSteps {
 		before := ""
 		if i > 0 {
@@ -1378,23 +1391,78 @@ func (ot *OptTester) OptStepsWeb() (string, error) {
 			return "", err
 		}
 		diffStr = strings.TrimRight(diffStr, " \r\t\n")
-		normDiff.WriteString(diffStr)
-		normDiff.WriteString("\n")
+		buf.WriteString(diffStr)
+		buf.WriteString("\n")
 	}
-	url, err := ot.encodeOptstepsURL(normDiff.String())
-	if err != nil {
-		return "", err
+	return buf.String(), nil
+}
+
+// optStepsExploreDiff produces the exploration steps as a diff where each new
+// expression is shown as a pair of "files" (showing the before and after
+// expression). Note that normalization rules that are applied as part of
+// creating the new expression are not shown separately.
+func (ot *OptTester) optStepsExploreDiff() (string, error) {
+	et := newExploreTracer(ot)
+
+	var buf bytes.Buffer
+
+	for step := 0; ; step++ {
+		if step > 2000 {
+			ot.output("step limit reached\n")
+			break
+		}
+		err := et.Next()
+		if err != nil {
+			return "", err
+		}
+		if et.Done() {
+			break
+		}
+
+		if ot.Flags.ExploreTraceRule != opt.InvalidRuleName &&
+			et.LastRuleName() != ot.Flags.ExploreTraceRule {
+			continue
+		}
+		newNodes := et.NewExprs()
+		before := et.fo.o.FormatExpr(et.SrcExpr(), ot.Flags.ExprFormat)
+
+		for i := range newNodes {
+			name := et.LastRuleName().String()
+			after := memo.FormatExpr(newNodes[i], ot.Flags.ExprFormat, et.fo.o.Memo(), ot.catalog)
+
+			diff := difflib.UnifiedDiff{
+				A:        difflib.SplitLines(before),
+				FromFile: fmt.Sprintf("a/%s", name),
+				B:        difflib.SplitLines(after),
+				ToFile:   fmt.Sprintf("b/%s", name),
+				Context:  10000,
+			}
+			diffStr, err := difflib.GetUnifiedDiffString(diff)
+			if err != nil {
+				return "", err
+			}
+			diffStr = strings.TrimRight(diffStr, " \r\t\n")
+			if diffStr == "" {
+				// It's possible that the "new" expression is identical to the original
+				// one; ignore it in that case.
+				continue
+			}
+			buf.WriteString(diffStr)
+			buf.WriteString("\n")
+		}
 	}
-	return url.String(), nil
+	return buf.String(), nil
 }
 
-func (ot *OptTester) encodeOptstepsURL(normDiff string) (url.URL, error) {
+func (ot *OptTester) encodeOptstepsURL(normDiff, exploreDiff string) (url.URL, error) {
 	output := struct {
-		SQL      string
-		Normdiff string
+		SQL         string
+		NormDiff    string
+		ExploreDiff string
 	}{
-		SQL:      ot.sql,
-		Normdiff: normDiff,
+		SQL:         ot.sql,
+		NormDiff:    normDiff,
+		ExploreDiff: exploreDiff,
 	}
 
 	var buf bytes.Buffer