fix: do not hit API Server for CSA preview

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## Unreleased
 
+- fix: ensure CSA does not hit API Server for preview (https://github.com/pulumi/pulumi-kubernetes/pull/2522)
+
 ## 4.0.3 (July 21, 2023)
 
 - fix: ensure data is not dropped when normalizing Secrets (https://github.com/pulumi/pulumi-kubernetes/pull/2514)

provider/pkg/provider/provider.go

@@ -1740,7 +1740,8 @@ func (k *kubeProvider) Create(
 	// 1: The input values contain unknowns
 	// 2: The resource GVK does not exist
 	// 3: The resource is a Patch resource
-	skipPreview := hasComputedValue(newInputs) || !k.gvkExists(newInputs) || isPatchURN(urn)
+	// 4: We are in client-side-apply mode
+	skipPreview := hasComputedValue(newInputs) || !k.gvkExists(newInputs) || isPatchURN(urn) || !k.serverSideApplyMode
 	// If this is a preview and the input meets one of the skip criteria, then return them as-is. This is compatible
 	// with prior behavior implemented by the Pulumi engine.
 	if req.GetPreview() && skipPreview {
@@ -1806,6 +1807,11 @@ func (k *kubeProvider) Create(
 	initialized, awaitErr := await.Creation(config)
 	if awaitErr != nil {
 		if req.GetPreview() {
+			if apierrors.IsForbidden(awaitErr) {
+				logger.V(1).Infof("unable to compute Server-side dry-run and defaulting to client-side: %s", awaitErr)
+				return &pulumirpc.CreateResponse{Id: "", Properties: req.GetProperties()}, nil
+			}
+
 			failedPreview := false
 			_, isPreviewErr := awaitErr.(await.PreviewError)
 			if k.isDryRunDisabledError(awaitErr) || isPreviewErr {

tests/sdk/nodejs/nodejs_test.go

@@ -1274,95 +1274,6 @@ func TestServerSideApplyEmptyMaps(t *testing.T) {
 	assert.Contains(t, string(out), "bar") // ConfigMap should have been updated with label foo=bar.
 }
 
-// TestServerSideApplyPreview tests that we automatically fall back to a client-side preview if the user does not have
-// permission to run the "patch" operation used by SSA.
-func TestServerSideApplyPreview(t *testing.T) {
-	manifestFile := filepath.Join("server-side-apply-preview", "manifest.yaml")
-
-	applyStep := baseOptions.With(integration.ProgramTestOptions{
-		Dir: filepath.Join("server-side-apply-preview", "step1"),
-		//ExpectRefreshChanges: true,
-		Quick: true,
-		OrderedConfig: []integration.ConfigValue{
-			{
-				Key:   "pulumi:disable-default-providers[0]",
-				Value: "kubernetes",
-				Path:  true,
-			},
-		},
-		ExtraRuntimeValidation: func(t *testing.T, stackInfo integration.RuntimeValidationStackInfo) {
-			cm := stackInfo.Outputs["cm"].(map[string]interface{})
-
-			// Validate we applied ConfigMap with expected data.
-			dataKeyV, ok, err := unstructured.NestedString(cm, "data", "dataKey")
-			assert.True(t, ok)
-			assert.NoError(t, err)
-			assert.Equal(t, "fake data", dataKeyV)
-		},
-		EditDirs: []integration.EditDir{
-			{
-				Dir:      filepath.Join("server-side-apply-preview", "step2"),
-				Additive: true,
-			},
-			{
-				Dir:           filepath.Join("server-side-apply-preview", "step3"),
-				Additive:      true,
-				ExpectFailure: true, // Update will fail with restricted permissions, but preview should still succeed.
-			},
-			{
-				Dir:      filepath.Join("server-side-apply-preview", "step4"),
-				Additive: true,
-			},
-		},
-	})
-
-	// Create a ServiceAccount and ClusterRoleBinding with "view"-only permission.
-	out, err := exec.Command("kubectl", "apply", "--filename", manifestFile).CombinedOutput()
-	assert.NoError(t, err)
-	assert.Contains(t, string(out), `serviceaccount/kubeconfig-sa created`)                           // Ensure ServiceAccount was created.
-	assert.Contains(t, string(out), `secret/sa-token created`)                                        // Ensure ServiceAccount token was created.
-	assert.Contains(t, string(out), `clusterrolebinding.rbac.authorization.k8s.io/view-only created`) // Ensure ClusterRoleBinding was created.
-
-	// Create a new kubeconfig context using the new "kubeconfig-sa" ServiceAccount.
-	newContextCmd := `kubectl config set-context kubeconfig-sa --user kubeconfig-sa --cluster $(kubectl config view -o jsonpath='{.clusters[0].name}')`
-	out, err = exec.Command("bash", "-c", newContextCmd).CombinedOutput()
-	assert.NoError(t, err)
-	assert.Contains(t, string(out), `Context "kubeconfig-sa" created`) // Ensure context was created.
-
-	// Create a new kubeconfig user using the new "kubeconfig-sa" ServiceAccount token.
-	newUserCmd := `kubectl config set-credentials kubeconfig-sa --token=$(kubectl -n kube-system get secret sa-token -o jsonpath='{.data.token}' | base64 --decode)`
-	out, err = exec.Command("bash", "-c", newUserCmd).CombinedOutput()
-	assert.NoError(t, err)
-	assert.Contains(t, string(out), `User "kubeconfig-sa" set.`) // Ensure user was created.
-
-	t.Log("created 'kubeconfig-sa' context")
-
-	// Register cleanup to undo kubeconfig and related resource changes.
-	t.Cleanup(func() {
-		// Clean up kubeconfig user.
-		out, err = exec.Command("kubectl", "config", "unset", "users.kubeconfig-sa").CombinedOutput()
-		assert.NoError(t, err)
-		assert.Contains(t, string(out), `"users.kubeconfig-sa" unset`)
-
-		// Clean up kubeconfig context.
-		out, err = exec.Command("kubectl", "config", "unset", "contexts.kubeconfig-sa").CombinedOutput()
-		assert.NoError(t, err)
-		assert.Contains(t, string(out), `"contexts.kubeconfig-sa" unset`)
-
-		// Clean up ServiceAccount and ClusterRoleBinding.
-		out, err = exec.Command("kubectl", "delete", "--filename", manifestFile).CombinedOutput()
-		assert.Contains(t, string(out), `serviceaccount "kubeconfig-sa" deleted`)
-		assert.Contains(t, string(out), `clusterrolebinding.rbac.authorization.k8s.io "view-only" deleted`)
-
-		t.Log("finished cleaning up 'kubeconfig-sa' context")
-	})
-
-	// Run program to update the Provider context and show expected changes to the ConfigMap.
-	pt := integration.ProgramTestManualLifeCycle(t, &applyStep)
-	err = pt.TestLifeCycleInitAndDestroy()
-	assert.NoError(t, err)
-}
-
 func TestServerSideApplyUpgrade(t *testing.T) {
 	test := baseOptions.With(integration.ProgramTestOptions{
 		Dir:                  filepath.Join("server-side-apply-upgrade", "step1"),

tests/sdk/nodejs/preview-auth/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: preview-auth-tests
+description: Tests SSA previews with restricted auth.
+runtime: nodejs

tests/sdk/nodejs/preview-auth/index.ts

@@ -0,0 +1,25 @@
+// Copyright 2016-2023, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as k8s from "@pulumi/kubernetes";
+
+// Create provider with SSA enabled.
+const provider = new k8s.Provider("k8s", {enableServerSideApply: true});
+
+new k8s.core.v1.ConfigMap("test", {
+    metadata: {
+        name: "foo",
+    },
+    data: {foo: "bar"},
+}, {provider});

tests/sdk/nodejs/server-side-apply-preview/step1/package.json → tests/sdk/nodejs/preview-auth/package.json

@@ -1,8 +1,9 @@
 {
-    "name": "server-side-apply-preview",
+    "name": "preview-auth",
     "version": "0.1.0",
     "dependencies": {
-        "@pulumi/pulumi": "latest"
+        "@pulumi/pulumi": "latest",
+        "@pulumi/random": "latest"
     },
     "peerDependencies": {
         "@pulumi/kubernetes": "latest"

tests/sdk/nodejs/preview-auth/service-account.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ci-robot # Name of service account to create.
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: limited-configmap-access
+rules:
+- apiGroups: [""]
+  resources: ["pods"] # Only grant access to pods, not configmaps.
+  verbs: ["get", "watch", "list", "create", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: restrict-configmap-access-binding
+subjects:
+- kind: ServiceAccount
+  name: ci-robot
+roleRef:
+  kind: Role
+  name: limited-configmap-access
+  apiGroup: rbac.authorization.k8s.io

tests/sdk/nodejs/preview_test.go

@@ -0,0 +1,155 @@
+// Copyright 2016-2023, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// nolint:goconst
+package test
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+)
+
+// TestPreview tests the `pulumi preview` CUJ with a serviceaccount that is not allowed to create a configmap. This test ensures that
+// we do a client side diff if the server dry run fails and do not error on preview. We also ensure that we error on pulumi up, and that
+// the configmap was not created using kubectl.
+func TestPreview(t *testing.T) {
+	test := baseOptions.With(integration.ProgramTestOptions{
+		Dir:                  "preview-auth",
+		ExpectRefreshChanges: true,
+		// Enable destroy-on-cleanup so we can shell out to kubectl to make external changes to the resource and reuse the same stack.
+		DestroyOnCleanup: true,
+		Quick:            true,
+		OrderedConfig: []integration.ConfigValue{
+			{
+				Key:   "pulumi:disable-default-providers[0]",
+				Value: "kubernetes",
+				Path:  true,
+			},
+		},
+	})
+
+	// Create service account and RBAC policies for the service account.
+	out, err := kubectl("apply -f preview-auth/service-account.yaml")
+	if err != nil {
+		t.Fatalf("unable to create RBAC policies: %s, out: %s", err, string(out))
+	}
+	t.Cleanup(func() {
+		log.Println("Deleting service-account and rbac")
+		kubectl("delete -f preview-auth/service-account.yaml")
+	})
+
+	// Create kubeconfig for service account.
+	kubeconfigPath, err := createSAKubeconfig(t, "ci-robot")
+	if err != nil {
+		t.Fatalf("unable to create kubeconfig: %s", err)
+	}
+	// Set kubeconfig env var for the test to use.
+	test = test.With(integration.ProgramTestOptions{
+		Env: []string{
+			"KUBECONFIG=" + kubeconfigPath,
+		},
+	})
+
+	// Initialize and the test project.
+	pt := integration.ProgramTestManualLifeCycle(t, &test)
+	err = pt.TestLifeCyclePrepare()
+	if err != nil {
+		t.Fatalf("unable to create temp dir: %s", err)
+	}
+	t.Cleanup(pt.TestCleanUp)
+	err = pt.TestLifeCycleInitialize()
+	if err != nil {
+		t.Fatalf("unable to init test project: %s", err)
+	}
+	t.Cleanup(func() {
+		destroyErr := pt.TestLifeCycleDestroy()
+		assert.NoError(t, destroyErr)
+	})
+
+	// Run a preview and assert no error, since we do a client side diff if server dry run fails.
+	err = pt.RunPulumiCommand("preview", "--non-interactive", "--diff", "--refresh", "--show-config")
+	assert.NoError(t, err)
+
+	// Run pulumi up and assert error since our SA doesn't have permissions to create a configmap.
+	err = pt.TestPreviewUpdateAndEdits()
+	assert.Error(t, err)
+
+	// Check that the configmap was not created using kubectl.
+	out, err = kubectl("get configmap foo")
+	assert.Error(t, err)
+	assert.Contains(t, string(out), `Error from server (NotFound): configmaps "foo" not found`)
+}
+
+// createSAKubeconfig creates a modified kubeconfig for the service account. The kubeconfig is created in a
+// tmp dir which is cleaned up after the test.
+func createSAKubeconfig(t *testing.T, saName string) (string, error) {
+	t.Helper()
+
+	// Create token to use for the service account.
+	token, err := kubectl(fmt.Sprintf("create token %s --duration=1h", saName))
+	if err != nil {
+		return "", err
+	}
+
+	// Load default kubeconfig as base.
+	config, err := clientcmd.NewDefaultClientConfigLoadingRules().Load()
+	if err != nil {
+		return "", err
+	}
+
+	// Use current-context cluster as base for service account context/auth.
+	config.AuthInfos[saName] = &clientcmdapi.AuthInfo{Token: string(token)}
+	config.Contexts[saName] = &clientcmdapi.Context{
+		Cluster:  config.Contexts[config.CurrentContext].Cluster,
+		AuthInfo: saName,
+	}
+	config.CurrentContext = saName
+
+	// Create tmp dir to store kubeconfig.
+	tmpDir, err := os.MkdirTemp("", "kubeconfig-preview")
+	if err != nil {
+		return "", err
+	}
+
+	t.Cleanup(func() {
+		log.Println("Deleting kubeconfig tmp dir")
+		assert.NoError(t, os.RemoveAll(tmpDir))
+	})
+
+	// Write kubeconfig to tmp dir.
+	kubeconfigPath := filepath.Join(tmpDir, "kubeconfig.txt")
+	err = clientcmd.WriteToFile(*config, kubeconfigPath)
+
+	return kubeconfigPath, err
+}
+
+// kubectl is a helper function to shell out and run kubectl commands.
+func kubectl(args ...string) ([]byte, error) {
+	var fmtArgs []string
+	for _, arg := range args {
+		fmtArgs = append(fmtArgs, strings.Fields(arg)...)
+	}
+
+	return exec.Command("kubectl", fmtArgs...).CombinedOutput()
+}