Allocate API key during org Setup

cmd/orgadm/main.go

@@ -5,10 +5,12 @@ import (
 	"flag"
 	"log"
 
+	apikeys "cloud.google.com/go/apikeys/apiv2"
 	secretmanager "cloud.google.com/go/secretmanager/apiv1"
 	"github.com/m-lab/autojoin/internal/adminx"
 	"github.com/m-lab/autojoin/internal/adminx/crmiface"
 	"github.com/m-lab/autojoin/internal/adminx/iamiface"
+	"github.com/m-lab/autojoin/internal/adminx/keysiface"
 	"github.com/m-lab/autojoin/internal/dnsname"
 	"github.com/m-lab/autojoin/internal/dnsx"
 	"github.com/m-lab/autojoin/internal/dnsx/dnsiface"
@@ -19,13 +21,15 @@ import (
 )
 
 var (
-	org     string
-	project string
+	org          string
+	project      string
+	updateTables bool
 )
 
 func init() {
 	flag.StringVar(&org, "org", "", "Organization name. Must match name assigned by M-Lab")
 	flag.StringVar(&project, "project", "", "GCP project to create organization resources")
+	flag.BoolVar(&updateTables, "update-tables", false, "Allow this org's service account to update table schemas")
 }
 
 func main() {
@@ -51,9 +55,13 @@ func main() {
 	ds, err := dns.NewService(ctx)
 	rtx.Must(err, "failed to create new dns service")
 	d := dnsx.NewManager(dnsiface.NewCloudDNSService(ds), project, dnsname.ProjectZone(project))
+	ac, err := apikeys.NewClient(ctx)
+	rtx.Must(err, "failed to create new apikey client")
+	k := adminx.NewAPIKeys(project, keysiface.NewKeys(ac), nn)
+	defer ac.Close()
 
-	o := adminx.NewOrg(project, crmiface.NewCRM(project, crm), sa, sm, d)
-	err = o.Setup(ctx, org)
+	o := adminx.NewOrg(project, crmiface.NewCRM(project, crm), sa, sm, d, k, updateTables)
+	key, err := o.Setup(ctx, org)
 	rtx.Must(err, "failed to set up new organization: "+org)
-	log.Println("okay")
+	log.Println("Setup okay - org:", org, "key:", key)
 }

internal/adminx/org.go

@@ -19,8 +19,14 @@ var (
 		` resource.name.startsWith("projects/_/buckets/staging-%s/objects/autoload/v2/%s")`)
 	// Restrict reads to the archive bucket. Needed so nodes can read jostler schemas.
 	expReadFmt = (`resource.name.startsWith("projects/_/buckets/archive-%s") ||` +
-		` resource.name.startsWith("projects/_/buckets/downloader-%s")` +
-		` resource.name.startsWith("projects/_/buckets/staging-%s") ||`)
+		` resource.name.startsWith("projects/_/buckets/downloader-%s") ||` +
+		` resource.name.startsWith("projects/_/buckets/staging-%s")`)
+
+	// Allow uploads to include tables. Needed for the authoritative schema update path.
+	expUploadTablesFmt = (`resource.name.startsWith("projects/_/buckets/archive-%s/objects/autoload/v2/%s") ||` +
+		` resource.name.startsWith("projects/_/buckets/staging-%s/objects/autoload/v2/%s") ||` +
+		` resource.name.startsWith("projects/_/buckets/archive-%s/objects/autoload/v2/tables") ||` +
+		` resource.name.startsWith("projects/_/buckets/staging-%s/objects/autoload/v2/tables")`)
 )
 
 // DNS is a simplified interface to the Google Cloud DNS API.
@@ -35,44 +41,57 @@ type CRM interface {
 	SetIamPolicy(ctx context.Context, req *cloudresourcemanager.SetIamPolicyRequest) error
 }
 
+// Keys is the interface used to manage organization API keys.
+type Keys interface {
+	CreateKey(ctx context.Context, org string) (string, error)
+}
+
 // Org contains fields needed to setup a new organization for Autojoined nodes.
 type Org struct {
-	Project string
-	crm     CRM
-	sam     *ServiceAccountsManager
-	sm      *SecretManager
-	dns     DNS
+	Project      string
+	crm          CRM
+	sam          *ServiceAccountsManager
+	sm           *SecretManager
+	dns          DNS
+	keys         Keys
+	updateTables bool
 }
 
 // NewOrg creates a new Org instance for setting up a new organization.
-func NewOrg(project string, crm CRM, sam *ServiceAccountsManager, sm *SecretManager, dns DNS) *Org {
+func NewOrg(project string, crm CRM, sam *ServiceAccountsManager, sm *SecretManager, dns DNS, k Keys, updateTables bool) *Org {
 	return &Org{
-		Project: project,
-		crm:     crm,
-		sam:     sam,
-		sm:      sm,
-		dns:     dns,
+		Project:      project,
+		crm:          crm,
+		sam:          sam,
+		sm:           sm,
+		dns:          dns,
+		keys:         k,
+		updateTables: updateTables,
 	}
 }
 
 // Setup should be run once on org creation to create all Google Cloud resources needed by the Autojoin API.
-func (o *Org) Setup(ctx context.Context, org string) error {
+func (o *Org) Setup(ctx context.Context, org string) (string, error) {
 	// Create service account with no keys.
 	sa, err := o.sam.CreateServiceAccount(ctx, org)
 	if err != nil {
-		return err
+		return "", err
 	}
-	err = o.ApplyPolicy(ctx, org, sa)
+	err = o.ApplyPolicy(ctx, org, sa, o.updateTables)
 	if err != nil {
-		return err
+		return "", err
 	}
 	// Create secret with no versions.
 	err = o.sm.CreateSecret(ctx, org)
 	if err != nil {
-		return err
+		return "", err
 	}
 	// Create DNS zone and zone split.
-	return o.RegisterDNS(ctx, org)
+	err = o.RegisterDNS(ctx, org)
+	if err != nil {
+		return "", err
+	}
+	return o.keys.CreateKey(ctx, org)
 }
 
 // RegisterDNS creates the organization zone and the zone split within the project zone.
@@ -99,7 +118,7 @@ func (o *Org) RegisterDNS(ctx context.Context, org string) error {
 
 // ApplyPolicy adds write restrictions for shared GCS buckets.
 // NOTE: By operating on project IAM policies, this method modifies project wide state.
-func (o *Org) ApplyPolicy(ctx context.Context, org string, account *iam.ServiceAccount) error {
+func (o *Org) ApplyPolicy(ctx context.Context, org string, account *iam.ServiceAccount, updateTables bool) error {
 	// Get current policy.
 	req := &cloudresourcemanager.GetIamPolicyRequest{
 		Options: &cloudresourcemanager.GetPolicyOptions{
@@ -111,20 +130,31 @@ func (o *Org) ApplyPolicy(ctx context.Context, org string, account *iam.ServiceA
 		log.Println("get policy", err)
 		return err
 	}
+	expression := ""
+	role := ""
+	if updateTables {
+		// Allow this role to upload data and update schema tables.
+		expression = fmt.Sprintf(expUploadTablesFmt, o.Project, org, o.Project, org, o.Project, o.Project)
+		role = "roles/storage.objectUser"
+	} else {
+		// Only allow this role to upload data.
+		expression = fmt.Sprintf(expUploadFmt, o.Project, org, o.Project, org)
+		role = "roles/storage.objectCreator"
+	}
 	// Setup new bindings.
 	bindings := []*cloudresourcemanager.Binding{
 		{
 			Condition: &cloudresourcemanager.Expr{
 				Title:      "Upload restriction for " + org,
-				Expression: fmt.Sprintf(expUploadFmt, o.Project, org, o.Project, org),
+				Expression: expression,
 			},
 			Members: []string{"serviceAccount:" + account.Email},
-			Role:    "roles/storage.objectCreator",
+			Role:    role,
 		},
 		{
 			Condition: &cloudresourcemanager.Expr{
 				Title:      "Read restriction for " + org,
-				Expression: fmt.Sprintf(expReadFmt, o.Project, o.Project),
+				Expression: fmt.Sprintf(expReadFmt, o.Project, o.Project, o.Project),
 			},
 			Members: []string{"serviceAccount:" + account.Email},
 			Role:    "roles/storage.objectViewer",

internal/adminx/org_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"strings"
 	"testing"
 
 	"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
@@ -24,6 +25,7 @@ type fakeCRM struct {
 	getPolicyErr error
 	setPolicyErr error
 	bindingCount int
+	policy       *cloudresourcemanager.Policy
 }
 
 func (f *fakeCRM) GetIamPolicy(ctx context.Context, req *cloudresourcemanager.GetIamPolicyRequest) (*cloudresourcemanager.Policy, error) {
@@ -32,6 +34,7 @@ func (f *fakeCRM) GetIamPolicy(ctx context.Context, req *cloudresourcemanager.Ge
 
 func (f *fakeCRM) SetIamPolicy(ctx context.Context, req *cloudresourcemanager.SetIamPolicyRequest) error {
 	f.bindingCount = len(req.Policy.Bindings)
+	f.policy = req.Policy
 	return f.setPolicyErr
 }
 
@@ -50,16 +53,28 @@ func (f *fakeDNS) RegisterZoneSplit(ctx context.Context, zone *dns.ManagedZone)
 	return f.regSplit, f.regSplitErr
 }
 
+type fakeAPIKeys struct {
+	createKey    string
+	createKeyErr error
+}
+
+func (f *fakeAPIKeys) CreateKey(ctx context.Context, org string) (string, error) {
+	return f.createKey, f.createKeyErr
+}
+
 func TestOrg_Setup(t *testing.T) {
 	tests := []struct {
-		name    string
-		project string
-		crm     CRM
-		sam     IAMService
-		smc     SecretManagerClient
-		dns     DNS
-		org     string
-		wantErr bool
+		name         string
+		project      string
+		crm          *fakeCRM
+		sam          IAMService
+		smc          SecretManagerClient
+		dns          DNS
+		org          string
+		keys         Keys
+		updateTables bool
+		bindingCount int
+		wantErr      bool
 	}{
 		{
 			name: "success",
@@ -87,6 +102,10 @@ func TestOrg_Setup(t *testing.T) {
 					DnsName: dnsname.OrgDNS("foo", "mlab-foo"),
 				},
 			},
+			keys: &fakeAPIKeys{
+				createKey: "this-is-a-fake-key",
+			},
+			bindingCount: 3,
 		},
 		{
 			name: "error-register-zone",
@@ -176,6 +195,10 @@ func TestOrg_Setup(t *testing.T) {
 					DnsName: dnsname.OrgDNS("foo", "mlab-foo"),
 				},
 			},
+			keys: &fakeAPIKeys{
+				createKey: "this-is-a-fake-key",
+			},
+			bindingCount: 3,
 		},
 		{
 			name: "error-create-service-account",
@@ -238,16 +261,65 @@ func TestOrg_Setup(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "success-update-tables-policy",
+			crm: &fakeCRM{
+				getPolicy: &cloudresourcemanager.Policy{
+					Bindings: []*cloudresourcemanager.Binding{
+						{
+							Members: []string{"foo"},
+							Role:    "roles/fooWriter",
+						},
+					},
+				},
+			},
+			sam: &fakeIAMService{
+				getAcct: &iam.ServiceAccount{
+					Name: "foo",
+				},
+			},
+			smc: &fakeSMC{
+				getSec: &secretmanagerpb.Secret{Name: "okay"},
+			},
+			dns: &fakeDNS{
+				regZone: &dns.ManagedZone{
+					Name:    dnsname.OrgZone("foo", "mlab-foo"),
+					DnsName: dnsname.OrgDNS("foo", "mlab-foo"),
+				},
+			},
+			keys: &fakeAPIKeys{
+				createKey: "this-is-a-fake-key",
+			},
+			updateTables: true,
+			bindingCount: 3,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			n := NewNamer("mlab-foo")
 			sam := NewServiceAccountsManager(tt.sam, n)
 			sm := NewSecretManager(tt.smc, n, sam)
-			o := NewOrg("mlab-foo", tt.crm, sam, sm, tt.dns)
-			if err := o.Setup(context.Background(), "foobar"); (err != nil) != tt.wantErr {
+			o := NewOrg("mlab-foo", tt.crm, sam, sm, tt.dns, tt.keys, tt.updateTables)
+			if _, err := o.Setup(context.Background(), "foobar"); (err != nil) != tt.wantErr {
 				t.Errorf("Org.Setup() error = %v, wantErr %v", err, tt.wantErr)
 			}
+			if !tt.wantErr && tt.crm != nil && tt.crm.bindingCount != tt.bindingCount {
+				t.Errorf("Org.Setup() failed to count bindings = %d, want %d", tt.crm.bindingCount, tt.bindingCount)
+			}
+			if tt.wantErr {
+				return
+			}
+			foundTables := false
+			for _, binding := range tt.crm.policy.Bindings {
+				if binding.Condition != nil {
+					if strings.Contains(binding.Condition.Expression, "tables") {
+						foundTables = true
+					}
+				}
+			}
+			if foundTables != tt.updateTables {
+				t.Errorf("Org.Setup() failed to update tables correctly = %t, want %t", foundTables, tt.updateTables)
+			}
 		})
 	}
 }