diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f14bf32b22..7f5de81c9d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -175,30 +175,28 @@ jobs: docker --version google-chrome --version && which google-chrome && chromedriver --version && which chromedriver timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9999)" != "200" ]]; do sleep 5; done' || false - - run: - name: Checkout Tests - command: git clone --single-branch --branch 2.0-Beta https://github.com/bonitoo-io/selenium-accept-infl2.git - run: name: Selenium tests command: | set +e - cd selenium-accept-infl2 + cd e2e npm install + sed -i "s/\"headless\": false/\"headless\": true/g" e2e.conf.json npm test; TEST_RESULT=$? npm run report:html npm run report:junit - mkdir -p ~/test-results/cucumber - mkdir -p ~/artifacts/html - cp ~/project/selenium-accept-infl2/report/cucumber_report.html ~/artifacts/html/cucumber_report.html - cp ~/project/selenium-accept-infl2/report/cucumber_junit.xml ~/test-results/cucumber/report.xml - cp ~/project/selenium-accept-infl2/report/cucumber_junit.xml ~/artifacts/report.xml - cp -r ~/project/selenium-accept-infl2/screenshots ~/artifacts + mkdir -p ~/e2e/test-results/cucumber + mkdir -p ~/e2e/artifacts/html + cp ~/project/e2e/report/cucumber_report.html ~/e2e/artifacts/html/cucumber_report.html + cp ~/project/e2e/report/cucumber_junit.xml ~/e2e/test-results/cucumber/report.xml + cp ~/project/e2e/report/cucumber_junit.xml ~/e2e/artifacts/report.xml + cp -r ~/project/e2e/screenshots ~/e2e/artifacts ls -al exit $TEST_RESULT - store_test_results: - path: ~/test-results + path: ~/e2e/test-results - store_artifacts: - path: ~/artifacts + path: ~/e2e/artifacts jstest: docker: - image: circleci/golang:1.13-node-browsers diff --git a/CHANGELOG.md b/CHANGELOG.md index ea30d9eaa9c..a3850c0d27e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ 1. [18568](https://github.com/influxdata/influxdb/pull/18568): Add support for config files to influxd and any cli.NewCommand use case 1. [18573](https://github.com/influxdata/influxdb/pull/18573): Extend influx stacks cmd with new influx stacks update cmd 1. [18581](https://github.com/influxdata/influxdb/pull/18581): Cache dashboard cell query results to use as a reference for cell configurations +1. [18595](https://github.com/influxdata/influxdb/pull/18595): Add ability to skip resources in a template by kind or by metadata.name +1. [18600](https://github.com/influxdata/influxdb/pull/18600): Extend influx apply with resource filter capabilities +1. [18601](https://github.com/influxdata/influxdb/pull/18601): Provide active config running influx config without args ## v2.0.0-beta.12 [2020-06-12] @@ -41,6 +44,7 @@ 1. [18361](https://github.com/influxdata/influxdb/pull/18361): Tokens list is now consistent with the other resource lists 1. [18346](https://github.com/influxdata/influxdb/pull/18346): Reduce the number of variables being hydrated when toggling variables 1. [18447](https://github.com/influxdata/influxdb/pull/18447): Redesign dashboard cell loading indicator to be more obvious +1. [18593](https://github.com/influxdata/influxdb/pull/18593): Add copyable User and Organization Ids to About page ## v2.0.0-beta.11 [2020-05-26] diff --git a/cmd/influx/config.go b/cmd/influx/config.go index ce2d09a1904..5a918fdfe8a 100644 --- a/cmd/influx/config.go +++ b/cmd/influx/config.go @@ -38,7 +38,7 @@ type cmdConfigBuilder struct { func (b *cmdConfigBuilder) cmd() *cobra.Command { cmd := b.newCmd("config [config name]", b.cmdSwitchActiveRunEFn, false) cmd.Short = "Config management commands" - cmd.Args = cobra.ExactArgs(1) + cmd.Args = cobra.ArbitraryArgs cmd.AddCommand( b.cmdCreate(), @@ -50,13 +50,35 @@ func (b *cmdConfigBuilder) cmd() *cobra.Command { } func (b *cmdConfigBuilder) cmdSwitchActiveRunEFn(cmd *cobra.Command, args []string) error { - cfg, err := b.svc.SwitchActive(args[0]) + if len(args) > 0 { + cfg, err := b.svc.SwitchActive(args[0]) + if err != nil { + return err + } + + return b.printConfigs(configPrintOpts{ + config: cfg, + }) + } + + configs, err := b.svc.ListConfigs() if err != nil { return err } + var active config.Config + for _, cfg := range configs { + if cfg.Active { + active = cfg + break + } + } + if !active.Active { + return nil + } + return b.printConfigs(configPrintOpts{ - config: cfg, + config: active, }) } diff --git a/cmd/influx/config_test.go b/cmd/influx/config_test.go index b7bb398c5cb..16e8cebf4e6 100644 --- a/cmd/influx/config_test.go +++ b/cmd/influx/config_test.go @@ -14,7 +14,6 @@ import ( ) func TestCmdConfig(t *testing.T) { - t.Run("create", func(t *testing.T) { tests := []struct { name string diff --git a/cmd/influx/pkg.go b/cmd/influx/pkg.go index 6ae3e4c5c3e..eae0b04961e 100644 --- a/cmd/influx/pkg.go +++ b/cmd/influx/pkg.go @@ -157,6 +157,17 @@ func (b *cmdPkgBuilder) cmdPkgApply() *cobra.Command { # Applying directories from many sources, file and URL influx apply -f $PATH_TO_TEMPLATE/template.yml -f $URL_TO_TEMPLATE + # Applying a template with actions to skip resources applied. The + # following example skips all buckets and the dashboard whose + # metadata.name field matches the provided $DASHBOARD_TMPL_NAME. + # format for filters: + # --filter=kind=Bucket + # --filter=resource=Label:$Label_TMPL_NAME + influx apply \ + -f $PATH_TO_TEMPLATE/template.yml \ + --filter kind=Bucket \ + --filter resource=Dashboard:$DASHBOARD_TMPL_NAME + For information about finding and using InfluxDB templates, see https://v2.docs.influxdata.com/v2.0/reference/cli/influx/apply/. @@ -174,6 +185,7 @@ func (b *cmdPkgBuilder) cmdPkgApply() *cobra.Command { b.applyOpts.secrets = []string{} cmd.Flags().StringSliceVar(&b.applyOpts.secrets, "secret", nil, "Secrets to provide alongside the template; format should --secret=SECRET_KEY=SECRET_VALUE --secret=SECRET_KEY_2=SECRET_VALUE_2") cmd.Flags().StringSliceVar(&b.applyOpts.envRefs, "env-ref", nil, "Environment references to provide alongside the template; format should --env-ref=REF_KEY=REF_VALUE --env-ref=REF_KEY_2=REF_VALUE_2") + cmd.Flags().StringSliceVar(&b.filters, "filter", nil, "Resources to skip when applying the template. Filter out by ‘kind’ or by ‘resource’") return cmd } @@ -220,6 +232,12 @@ func (b *cmdPkgBuilder) pkgApplyRunEFn(cmd *cobra.Command, args []string) error pkger.ApplyWithStackID(stackID), } + actionOpts, err := parseTemplateActions(b.filters) + if err != nil { + return err + } + opts = append(opts, actionOpts...) + dryRunImpact, err := svc.DryRun(context.Background(), influxOrgID, 0, opts...) if err != nil { return err @@ -266,6 +284,41 @@ func (b *cmdPkgBuilder) pkgApplyRunEFn(cmd *cobra.Command, args []string) error return nil } +func parseTemplateActions(args []string) ([]pkger.ApplyOptFn, error) { + var opts []pkger.ApplyOptFn + for _, rawAct := range args { + pair := strings.SplitN(rawAct, "=", 2) + if len(pair) < 2 { + continue + } + key, val := pair[0], pair[1] + switch strings.ToLower(key) { + case "kind": + opts = append(opts, pkger.ApplyWithKindSkip(pkger.ActionSkipKind{ + Kind: pkger.Kind(val), + })) + case "resource": + pp := strings.SplitN(val, ":", 2) + if len(pair) != 2 { + return nil, fmt.Errorf(`invalid skipResource action provided: %q; + Expected format --action=skipResource=Label:$LABEL_ID`, rawAct) + } + kind, metaName := pp[0], pp[1] + opts = append(opts, pkger.ApplyWithResourceSkip(pkger.ActionSkipResource{ + Kind: pkger.Kind(kind), + MetaName: metaName, + })) + default: + return nil, fmt.Errorf(`invalid action provided: %q; + Expected format --action=skipResource=Label:$LABEL_ID + or + Expected format --action=skipKind=Bucket`, rawAct) + } + } + + return opts, nil +} + func (b *cmdPkgBuilder) cmdPkgExport() *cobra.Command { cmd := b.newCmd("export", b.pkgExportRunEFn, true) cmd.Short = "Export existing resources as a package" @@ -383,14 +436,14 @@ func (b *cmdPkgBuilder) cmdPkgExportAll() *cobra.Command { influx pkg export all --org $ORG_NAME # Export all bucket resources - influx export all --org $ORG_NAME --filter=resourceKind=Bucket + influx export all --org $ORG_NAME --filter=kind=Bucket # Export all resources associated with label Foo influx export all --org $ORG_NAME --filter=labelName=Foo # Export all bucket resources and filter by label Foo influx export all --org $ORG_NAME \ - --filter=resourceKind=Bucket \ + --filter=kind=Bucket \ --filter=labelName=Foo # Export all bucket or dashboard resources and filter by label Foo. @@ -398,8 +451,8 @@ func (b *cmdPkgBuilder) cmdPkgExportAll() *cobra.Command { # This example will export a resource if it is a dashboard or # bucket and has an associated label of Foo. influx export all --org $ORG_NAME \ - --filter=resourceKind=Bucket \ - --filter=resourceKind=Dashboard \ + --filter=kind=Bucket \ + --filter=kind=Dashboard \ --filter=labelName=Foo For information about exporting InfluxDB templates, see @@ -439,7 +492,7 @@ func (b *cmdPkgBuilder) pkgExportAllRunEFn(cmd *cobra.Command, args []string) er switch key, val := pair[0], pair[1]; key { case "labelName": labelNames = append(labelNames, val) - case "resourceKind": + case "kind", "resourceKind": k := pkger.Kind(val) if err := k.OK(); err != nil { return err diff --git a/cmd/influxd/launcher/pkger_test.go b/cmd/influxd/launcher/pkger_test.go index 8002521c6ed..92a460b2dd4 100644 --- a/cmd/influxd/launcher/pkger_test.go +++ b/cmd/influxd/launcher/pkger_test.go @@ -346,7 +346,8 @@ func TestLauncher_Pkger(t *testing.T) { }) t.Run("that has already been deleted should be successful", func(t *testing.T) { - newStack, _ := newStackFn(t, pkger.Stack{}) + newStack, cleanup := newStackFn(t, pkger.Stack{}) + defer cleanup() err := svc.DeleteStack(ctx, struct{ OrgID, UserID, StackID influxdb.ID }{ OrgID: l.Org.ID, @@ -1537,6 +1538,140 @@ func TestLauncher_Pkger(t *testing.T) { }) }) + t.Run("apply with actions", func(t *testing.T) { + var ( + bucketPkgName = "rucketeer-1" + checkPkgName = "checkers" + dashPkgName = "dash-of-salt" + endpointPkgName = "endzo" + labelPkgName = "labelino" + rulePkgName = "oh-doyle-rules" + taskPkgName = "tap" + telegrafPkgName = "teletype" + variablePkgName = "laces-out-dan" + ) + + tests := []struct { + name string + applyOpts []pkger.ApplyOptFn + }{ + { + name: "skip resource", + applyOpts: []pkger.ApplyOptFn{ + pkger.ApplyWithResourceSkip(pkger.ActionSkipResource{ + Kind: pkger.KindBucket, + MetaName: bucketPkgName, + }), + pkger.ApplyWithResourceSkip(pkger.ActionSkipResource{ + Kind: pkger.KindCheckDeadman, + MetaName: checkPkgName, + }), + pkger.ApplyWithResourceSkip(pkger.ActionSkipResource{ + Kind: pkger.KindDashboard, + MetaName: dashPkgName, + }), + pkger.ApplyWithResourceSkip(pkger.ActionSkipResource{ + Kind: pkger.KindNotificationEndpointHTTP, + MetaName: endpointPkgName, + }), + pkger.ApplyWithResourceSkip(pkger.ActionSkipResource{ + Kind: pkger.KindLabel, + MetaName: labelPkgName, + }), + pkger.ApplyWithResourceSkip(pkger.ActionSkipResource{ + Kind: pkger.KindNotificationRule, + MetaName: rulePkgName, + }), + pkger.ApplyWithResourceSkip(pkger.ActionSkipResource{ + Kind: pkger.KindTask, + MetaName: taskPkgName, + }), + pkger.ApplyWithResourceSkip(pkger.ActionSkipResource{ + Kind: pkger.KindTelegraf, + MetaName: telegrafPkgName, + }), + pkger.ApplyWithResourceSkip(pkger.ActionSkipResource{ + Kind: pkger.KindVariable, + MetaName: variablePkgName, + }), + }, + }, + { + name: "skip kind", + applyOpts: []pkger.ApplyOptFn{ + pkger.ApplyWithKindSkip(pkger.ActionSkipKind{ + Kind: pkger.KindBucket, + }), + pkger.ApplyWithKindSkip(pkger.ActionSkipKind{ + Kind: pkger.KindCheckDeadman, + }), + pkger.ApplyWithKindSkip(pkger.ActionSkipKind{ + Kind: pkger.KindDashboard, + }), + pkger.ApplyWithKindSkip(pkger.ActionSkipKind{ + Kind: pkger.KindNotificationEndpointHTTP, + }), + pkger.ApplyWithKindSkip(pkger.ActionSkipKind{ + Kind: pkger.KindLabel, + }), + pkger.ApplyWithKindSkip(pkger.ActionSkipKind{ + Kind: pkger.KindNotificationRule, + }), + pkger.ApplyWithKindSkip(pkger.ActionSkipKind{ + Kind: pkger.KindTask, + }), + pkger.ApplyWithKindSkip(pkger.ActionSkipKind{ + Kind: pkger.KindTelegraf, + }), + pkger.ApplyWithKindSkip(pkger.ActionSkipKind{ + Kind: pkger.KindVariable, + }), + }, + }, + } + + for _, tt := range tests { + fn := func(t *testing.T) { + stack, cleanup := newStackFn(t, pkger.Stack{}) + defer cleanup() + + pkg := newPkg( + newBucketObject(bucketPkgName, "", ""), + newCheckDeadmanObject(t, checkPkgName, "", time.Hour), + newDashObject(dashPkgName, "", ""), + newEndpointHTTP(endpointPkgName, "", ""), + newLabelObject(labelPkgName, "", "", ""), + newRuleObject(t, rulePkgName, "", endpointPkgName, ""), + newTaskObject(taskPkgName, "", ""), + newTelegrafObject(telegrafPkgName, "", ""), + newVariableObject(variablePkgName, "", ""), + ) + + impact, err := svc.Apply(ctx, l.Org.ID, l.User.ID, + append( + tt.applyOpts, + pkger.ApplyWithPkg(pkg), + pkger.ApplyWithStackID(stack.ID), + )..., + ) + require.NoError(t, err) + + summary := impact.Summary + assert.Empty(t, summary.Buckets) + assert.Empty(t, summary.Checks) + assert.Empty(t, summary.Dashboards) + assert.Empty(t, summary.NotificationEndpoints) + assert.Empty(t, summary.Labels) + assert.Empty(t, summary.NotificationRules, 0) + assert.Empty(t, summary.Tasks) + assert.Empty(t, summary.TelegrafConfigs) + assert.Empty(t, summary.Variables) + } + + t.Run(tt.name, fn) + } + }) + t.Run("exporting the existing state of stack resources to a pkg", func(t *testing.T) { testStackApplyFn := func(t *testing.T) (pkger.Summary, pkger.Stack, func()) { t.Helper() @@ -1874,6 +2009,10 @@ func TestLauncher_Pkger(t *testing.T) { }) t.Run("errors incurred during application of package rolls back to state before package", func(t *testing.T) { + stacks, err := svc.ListStacks(ctx, l.Org.ID, pkger.ListFilter{}) + require.NoError(t, err) + require.Empty(t, stacks) + svc := pkger.NewService( pkger.WithBucketSVC(l.BucketService(t)), pkger.WithDashboardSVC(l.DashboardService(t)), @@ -1891,7 +2030,7 @@ func TestLauncher_Pkger(t *testing.T) { pkger.WithVariableSVC(l.VariableService(t)), ) - _, err := svc.Apply(ctx, l.Org.ID, l.User.ID, pkger.ApplyWithPkg(newCompletePkg(t))) + _, err = svc.Apply(ctx, l.Org.ID, l.User.ID, pkger.ApplyWithPkg(newCompletePkg(t))) require.Error(t, err) bkts, _, err := l.BucketService(t).FindBuckets(ctx, influxdb.BucketFilter{OrganizationID: &l.Org.ID}) @@ -1941,6 +2080,10 @@ func TestLauncher_Pkger(t *testing.T) { vars, err := l.VariableService(t).FindVariables(ctx, influxdb.VariableFilter{OrganizationID: &l.Org.ID}) require.NoError(t, err) assert.Empty(t, vars) + + stacks, err = svc.ListStacks(ctx, l.Org.ID, pkger.ListFilter{}) + require.NoError(t, err) + require.Empty(t, stacks) }) hasLabelAssociations := func(t *testing.T, associations []pkger.SummaryLabel, numAss int, expectedNames ...string) { diff --git a/http/query.go b/http/query.go index 11cd2b5ed41..466c6960b64 100644 --- a/http/query.go +++ b/http/query.go @@ -330,6 +330,7 @@ func QueryRequestFromProxyRequest(req *query.ProxyRequest) (*QueryRequest, error qr.Type = "flux" qr.Query = c.Query qr.Extern = c.Extern + qr.Now = c.Now case lang.ASTCompiler: qr.Type = "flux" qr.AST = c.AST diff --git a/http/query_test.go b/http/query_test.go index 5aafaf41cdc..3e7a243932b 100644 --- a/http/query_test.go +++ b/http/query_test.go @@ -610,3 +610,59 @@ func Test_decodeProxyQueryRequest(t *testing.T) { }) } } + +func TestProxyRequestToQueryRequest_Compilers(t *testing.T) { + tests := []struct { + name string + pr query.ProxyRequest + want QueryRequest + }{ + { + name: "flux compiler copied", + pr: query.ProxyRequest{ + Dialect: &query.NoContentDialect{}, + Request: query.Request{ + Compiler: lang.FluxCompiler{ + Query: `howdy`, + Now: time.Unix(45, 45), + }, + }, + }, + want: QueryRequest{ + Type: "flux", + Query: `howdy`, + PreferNoContent: true, + Now: time.Unix(45, 45), + }, + }, + { + name: "AST compiler copied", + pr: query.ProxyRequest{ + Dialect: &query.NoContentDialect{}, + Request: query.Request{ + Compiler: lang.ASTCompiler{ + Now: time.Unix(45, 45), + AST: mustMarshal(&ast.Package{}), + }, + }, + }, + want: QueryRequest{ + Type: "flux", + PreferNoContent: true, + AST: mustMarshal(&ast.Package{}), + Now: time.Unix(45, 45), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + got, err := QueryRequestFromProxyRequest( &tt.pr ) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(*got, tt.want) { + t.Errorf("QueryRequestFromProxyRequest = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/http/swagger.yml b/http/swagger.yml index 7a45262cbb6..4ce8420606a 100644 --- a/http/swagger.yml +++ b/http/swagger.yml @@ -4500,7 +4500,7 @@ paths: $ref: "#/components/schemas/Error" "/orgs/{orgID}/invites/{inviteID}/resend": post: - operationId: DeleteOrgsIDInviteID + operationId: PostOrgsIDInviteID tags: - Invites - Organizations @@ -7691,18 +7691,51 @@ components: contentType: type: string required: ["url"] - PkgCreateKind: + actions: + type: array + items: + oneOf: + - type: object + properties: + action: + type: string + enum: ["skipKind"] + properties: + type: object + properties: + kind: + $ref: "#/components/schemas/TemplateKind" + required: ["kind"] + - type: object + properties: + action: + type: string + enum: ["skipResource"] + properties: + type: object + properties: + kind: + $ref: "#/components/schemas/TemplateKind" + resourceTemplateName: + type: string + required: ["kind", "resourceTemplateName"] + TemplateKind: type: string enum: - - bucket - - check - - dashboard - - label - - notification_endpoint - - notification_rule - - task - - telegraf - - variable + - Bucket + - Check + - CheckDeadman + - CheckThreshold + - Dashboard + - Label + - NotificationEndpoint + - NotificationEndpointHTTP + - NotificationEndpointPagerDuty + - NotificationEndpointSlack + - NotificationRule + - Task + - Telegraf + - Variable PkgCreate: type: object properties: @@ -7723,14 +7756,14 @@ components: byResourceKind: type: array items: - $ref: "#/components/schemas/PkgCreateKind" + $ref: "#/components/schemas/TemplateKind" resources: type: object properties: id: type: string kind: - $ref: "#/components/schemas/PkgCreateKind" + $ref: "#/components/schemas/TemplateKind" name: type: string required: [id, kind] @@ -7742,20 +7775,7 @@ components: apiVersion: type: string kind: - type: string - enum: - - Bucket - - CheckDeadman - - CheckThreshold - - Dashboard - - Label - - NotificationEndpointHTTP - - NotificationEndpointPagerDuty - - NotificationEndpointSlack - - NotificationRule - - Task - - Telegraf - - Variable + $ref: "#/components/schemas/TemplateKind" meta: type: object properties: @@ -8351,7 +8371,7 @@ components: type: object properties: kind: - type: string + $ref: "#/components/schemas/TemplateKind" reason: type: string fields: @@ -8420,7 +8440,7 @@ components: resourceID: type: string kind: - type: string + $ref: "#/components/schemas/TemplateKind" pkgName: type: string associations: @@ -8429,7 +8449,7 @@ components: type: object properties: kind: - type: string + $ref: "#/components/schemas/TemplateKind" pkgName: type: string urls: diff --git a/pkger/http_remote_service.go b/pkger/http_remote_service.go index de50c4aef1d..815c6ff2e94 100644 --- a/pkger/http_remote_service.go +++ b/pkger/http_remote_service.go @@ -2,6 +2,7 @@ package pkger import ( "context" + "encoding/json" "net/http" "github.com/influxdata/influxdb/v2" @@ -214,6 +215,27 @@ func (s *HTTPRemoteService) apply(ctx context.Context, orgID influxdb.ID, dryRun reqBody.StackID = &stackID } + for act := range opt.ResourcesToSkip { + b, err := json.Marshal(act) + if err != nil { + return PkgImpactSummary{}, influxErr(influxdb.EInvalid, err) + } + reqBody.RawActions = append(reqBody.RawActions, ReqRawAction{ + Action: string(ActionTypeSkipResource), + Properties: b, + }) + } + for kind := range opt.KindsToSkip { + b, err := json.Marshal(ActionSkipKind{Kind: kind}) + if err != nil { + return PkgImpactSummary{}, influxErr(influxdb.EInvalid, err) + } + reqBody.RawActions = append(reqBody.RawActions, ReqRawAction{ + Action: string(ActionTypeSkipKind), + Properties: b, + }) + } + var resp RespApplyPkg err := s.Client. PostJSON(reqBody, RoutePrefix, "/apply"). diff --git a/pkger/http_server.go b/pkger/http_server.go index dc92f2c15b2..acadfb4de8f 100644 --- a/pkger/http_server.go +++ b/pkger/http_server.go @@ -14,6 +14,7 @@ import ( "github.com/go-chi/chi/middleware" "github.com/influxdata/influxdb/v2" pctx "github.com/influxdata/influxdb/v2/context" + ierrors "github.com/influxdata/influxdb/v2/kit/errors" kithttp "github.com/influxdata/influxdb/v2/kit/transport/http" "github.com/influxdata/influxdb/v2/pkg/jsonnet" "go.uber.org/zap" @@ -470,6 +471,13 @@ func (p ReqRawPkg) Encoding() Encoding { return convertEncoding(p.ContentType, source) } +// ReqRawAction is a raw action consumers can provide to change the behavior +// of the application of a template. +type ReqRawAction struct { + Action string `json:"action"` + Properties json.RawMessage `json:"properties"` +} + // ReqApplyPkg is the request body for a json or yaml body for the apply pkg endpoint. type ReqApplyPkg struct { DryRun bool `json:"dryRun" yaml:"dryRun"` @@ -489,6 +497,8 @@ type ReqApplyPkg struct { EnvRefs map[string]string `json:"envRefs"` Secrets map[string]string `json:"secrets"` + + RawActions []ReqRawAction `json:"actions"` } // Pkgs returns all pkgs associated with the request. @@ -545,6 +555,66 @@ func (r ReqApplyPkg) Pkgs(encoding Encoding) (*Pkg, error) { return Combine(rawPkgs, ValidWithoutResources(), ValidSkipParseError()) } +type actionType string + +// various ActionTypes the transport API speaks +const ( + ActionTypeSkipKind actionType = "skipKind" + ActionTypeSkipResource actionType = "skipResource" +) + +func (r ReqApplyPkg) validActions() (struct { + SkipKinds []ActionSkipKind + SkipResources []ActionSkipResource +}, error) { + type actions struct { + SkipKinds []ActionSkipKind + SkipResources []ActionSkipResource + } + + unmarshalErrFn := func(err error, idx int, actionType string) error { + msg := fmt.Sprintf("failed to unmarshal properties for actions[%d] %q", idx, actionType) + return ierrors.Wrap(err, msg) + } + + kindErrFn := func(err error, idx int, actionType string) error { + msg := fmt.Sprintf("invalid kind for actions[%d] %q", idx, actionType) + return ierrors.Wrap(err, msg) + } + + var out actions + for i, rawAct := range r.RawActions { + switch a := rawAct.Action; actionType(a) { + case ActionTypeSkipResource: + var asr ActionSkipResource + if err := json.Unmarshal(rawAct.Properties, &asr); err != nil { + return actions{}, influxErr(influxdb.EInvalid, unmarshalErrFn(err, i, a)) + } + if err := asr.Kind.OK(); err != nil { + return actions{}, influxErr(influxdb.EInvalid, kindErrFn(err, i, a)) + } + out.SkipResources = append(out.SkipResources, asr) + case ActionTypeSkipKind: + var ask ActionSkipKind + if err := json.Unmarshal(rawAct.Properties, &ask); err != nil { + return actions{}, influxErr(influxdb.EInvalid, unmarshalErrFn(err, i, a)) + } + if err := ask.Kind.OK(); err != nil { + return actions{}, influxErr(influxdb.EInvalid, kindErrFn(err, i, a)) + } + out.SkipKinds = append(out.SkipKinds, ask) + default: + msg := fmt.Sprintf( + "invalid action type %q provided for actions[%d] ; Must be one of [%s]", + a, i, ActionTypeSkipResource, + ) + return actions{}, influxErr(influxdb.EInvalid, msg) + } + } + + return out, nil +} + // RespApplyPkg is the response body for the apply pkg endpoint. type RespApplyPkg struct { Sources []string `json:"sources" yaml:"sources"` @@ -583,13 +653,6 @@ func (s *HTTPServer) applyPkg(w http.ResponseWriter, r *http.Request) { } } - auth, err := pctx.GetAuthorizer(r.Context()) - if err != nil { - s.api.Err(w, r, err) - return - } - userID := auth.GetUserID() - parsedPkg, err := reqBody.Pkgs(encoding) if err != nil { s.api.Err(w, r, &influxdb.Error{ @@ -599,11 +662,30 @@ func (s *HTTPServer) applyPkg(w http.ResponseWriter, r *http.Request) { return } + actions, err := reqBody.validActions() + if err != nil { + s.api.Err(w, r, err) + return + } + applyOpts := []ApplyOptFn{ ApplyWithEnvRefs(reqBody.EnvRefs), ApplyWithPkg(parsedPkg), ApplyWithStackID(stackID), } + for _, a := range actions.SkipResources { + applyOpts = append(applyOpts, ApplyWithResourceSkip(a)) + } + for _, a := range actions.SkipKinds { + applyOpts = append(applyOpts, ApplyWithKindSkip(a)) + } + + auth, err := pctx.GetAuthorizer(r.Context()) + if err != nil { + s.api.Err(w, r, err) + return + } + userID := auth.GetUserID() if reqBody.DryRun { impact, err := s.svc.DryRun(r.Context(), *orgID, userID, applyOpts...) diff --git a/pkger/main_test.go b/pkger/main_test.go new file mode 100644 index 00000000000..d2e0ef88f1e --- /dev/null +++ b/pkger/main_test.go @@ -0,0 +1,32 @@ +package pkger + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "testing" +) + +var ( + missedTemplateCacheCounter int64 + availableTemplateFiles = map[string][]byte{} +) + +func TestMain(m *testing.M) { + // this is to prime the files so we don't have to keep reading from disk for each test + // cuts runtime of tests down by 80% on current mac + files, _ := ioutil.ReadDir("testdata") + for _, f := range files { + relativeName := path.Join("testdata", f.Name()) + b, err := ioutil.ReadFile(relativeName) + if err == nil { + availableTemplateFiles[relativeName] = b + } + } + exitCode := m.Run() + if missedTemplateCacheCounter > 0 { + fmt.Println("templates that missed cache: ", missedTemplateCacheCounter) + } + os.Exit(exitCode) +} diff --git a/pkger/parser_test.go b/pkger/parser_test.go index 261052969ac..ded5c1b378a 100644 --- a/pkger/parser_test.go +++ b/pkger/parser_test.go @@ -1,6 +1,7 @@ package pkger import ( + "bytes" "errors" "fmt" "net/url" @@ -8,6 +9,7 @@ import ( "sort" "strconv" "strings" + "sync/atomic" "testing" "time" @@ -4321,7 +4323,16 @@ func nextField(t *testing.T, field string) (string, int) { func validParsedPkgFromFile(t *testing.T, path string, encoding Encoding) *Pkg { t.Helper() - pkg := newParsedPkg(t, FromFile(path), encoding) + var readFn ReaderFn + templateBytes, ok := availableTemplateFiles[path] + if ok { + readFn = FromReader(bytes.NewBuffer(templateBytes), "file://"+path) + } else { + readFn = FromFile(path) + atomic.AddInt64(&missedTemplateCacheCounter, 1) + } + + pkg := newParsedPkg(t, readFn, encoding) u := url.URL{ Scheme: "file", Path: path, diff --git a/pkger/service.go b/pkger/service.go index a2c67dacc92..608d9b4c1a0 100644 --- a/pkger/service.go +++ b/pkger/service.go @@ -295,7 +295,7 @@ func (s *Service) InitStack(ctx context.Context, userID influxdb.ID, stack Stack if _, err := s.orgSVC.FindOrganizationByID(ctx, stack.OrgID); err != nil { if influxdb.ErrorCode(err) == influxdb.ENotFound { msg := fmt.Sprintf("organization dependency does not exist for id[%q]", stack.OrgID.String()) - return Stack{}, toInfluxError(influxdb.EConflict, msg) + return Stack{}, influxErr(influxdb.EConflict, msg) } return Stack{}, internalErr(err) } @@ -963,7 +963,10 @@ func (s *Service) dryRun(ctx context.Context, orgID influxdb.ID, pkg *Pkg, opt A parseErr = err } - state := newStateCoordinator(pkg) + state := newStateCoordinator(pkg, resourceActions{ + skipKinds: opt.KindsToSkip, + skipResources: opt.ResourcesToSkip, + }) if opt.StackID > 0 { if err := s.addStackState(ctx, opt.StackID, state); err != nil { @@ -1364,16 +1367,33 @@ func (s *Service) addStackState(ctx context.Context, stackID influxdb.ID, state return nil } -// ApplyOpt is an option for applying a package. -type ApplyOpt struct { - Pkgs []*Pkg - EnvRefs map[string]string - MissingSecrets map[string]string - StackID influxdb.ID -} +type ( + // ApplyOpt is an option for applying a package. + ApplyOpt struct { + Pkgs []*Pkg + EnvRefs map[string]string + MissingSecrets map[string]string + StackID influxdb.ID + ResourcesToSkip map[ActionSkipResource]bool + KindsToSkip map[Kind]bool + } -// ApplyOptFn updates the ApplyOpt per the functional option. -type ApplyOptFn func(opt *ApplyOpt) + // ActionSkipResource provides an action from the consumer to use the pkg with + // modifications to the resource kind and pkg name that will be applied. + ActionSkipResource struct { + Kind Kind `json:"kind"` + MetaName string `json:"resourceTemplateName"` + } + + // ActionSkipKind provides an action from the consumer to use the pkg with + // modifications to the resource kinds will be applied. + ActionSkipKind struct { + Kind Kind `json:"kind"` + } + + // ApplyOptFn updates the ApplyOpt per the functional option. + ApplyOptFn func(opt *ApplyOpt) +) // ApplyWithEnvRefs provides env refs to saturate the missing reference fields in the pkg. func ApplyWithEnvRefs(envRefs map[string]string) ApplyOptFn { @@ -1389,6 +1409,42 @@ func ApplyWithPkg(pkg *Pkg) ApplyOptFn { } } +// ApplyWithResourceSkip provides an action skip a resource in the application of a template. +func ApplyWithResourceSkip(action ActionSkipResource) ApplyOptFn { + return func(opt *ApplyOpt) { + if opt.ResourcesToSkip == nil { + opt.ResourcesToSkip = make(map[ActionSkipResource]bool) + } + switch action.Kind { + case KindCheckDeadman, KindCheckThreshold: + action.Kind = KindCheck + case KindNotificationEndpointHTTP, + KindNotificationEndpointPagerDuty, + KindNotificationEndpointSlack: + action.Kind = KindNotificationEndpoint + } + opt.ResourcesToSkip[action] = true + } +} + +// ApplyWithKindSkip provides an action skip a kidn in the application of a template. +func ApplyWithKindSkip(action ActionSkipKind) ApplyOptFn { + return func(opt *ApplyOpt) { + if opt.KindsToSkip == nil { + opt.KindsToSkip = make(map[Kind]bool) + } + switch action.Kind { + case KindCheckDeadman, KindCheckThreshold: + action.Kind = KindCheck + case KindNotificationEndpointHTTP, + KindNotificationEndpointPagerDuty, + KindNotificationEndpointSlack: + action.Kind = KindNotificationEndpoint + } + opt.KindsToSkip[action.Kind] = true + } +} + // ApplyWithSecrets provides secrets to the platform that the pkg will need. func ApplyWithSecrets(secrets map[string]string) ApplyOptFn { return func(o *ApplyOpt) { @@ -1449,6 +1505,11 @@ func (s *Service) Apply(ctx context.Context, orgID, userID influxdb.ID, opts ... updateStackFn := s.updateStackAfterSuccess if e != nil { updateStackFn = s.updateStackAfterRollback + if opt.StackID == 0 { + if err := s.store.DeleteStack(ctx, stackID); err != nil { + s.log.Error("failed to delete created stack", zap.Error(err)) + } + } } err := updateStackFn(ctx, stackID, state, pkg.Sources()) @@ -3488,7 +3549,7 @@ func validURLs(urls []string) error { for _, u := range urls { if _, err := url.Parse(u); err != nil { msg := fmt.Sprintf("url invalid for entry %q", u) - return toInfluxError(influxdb.EInvalid, msg) + return influxErr(influxdb.EInvalid, msg) } } return nil @@ -3513,12 +3574,23 @@ func internalErr(err error) error { if err == nil { return nil } - return toInfluxError(influxdb.EInternal, err.Error()) + return influxErr(influxdb.EInternal, err) } -func toInfluxError(code string, msg string) *influxdb.Error { - return &influxdb.Error{ +func influxErr(code string, errArg interface{}, rest ...interface{}) *influxdb.Error { + err := &influxdb.Error{ Code: code, - Msg: msg, } + for _, a := range append(rest, errArg) { + switch v := a.(type) { + case string: + err.Msg = v + case error: + err.Err = v + case nil: + case interface{ String() string }: + err.Msg = v.String() + } + } + return err } diff --git a/pkger/service_models.go b/pkger/service_models.go index de5c7262694..c5d59d073f8 100644 --- a/pkger/service_models.go +++ b/pkger/service_models.go @@ -23,7 +23,7 @@ type stateCoordinator struct { labelMappingsToRemove []stateLabelMappingForRemoval } -func newStateCoordinator(pkg *Pkg) *stateCoordinator { +func newStateCoordinator(pkg *Pkg, acts resourceActions) *stateCoordinator { state := stateCoordinator{ mBuckets: make(map[string]*stateBucket), mChecks: make(map[string]*stateCheck), @@ -37,54 +37,81 @@ func newStateCoordinator(pkg *Pkg) *stateCoordinator { } for _, pkgBkt := range pkg.buckets() { + if acts.skipResource(KindBucket, pkgBkt.PkgName()) { + continue + } state.mBuckets[pkgBkt.PkgName()] = &stateBucket{ parserBkt: pkgBkt, stateStatus: StateStatusNew, } } for _, pkgCheck := range pkg.checks() { + if acts.skipResource(KindCheck, pkgCheck.PkgName()) { + continue + } state.mChecks[pkgCheck.PkgName()] = &stateCheck{ parserCheck: pkgCheck, stateStatus: StateStatusNew, } } for _, pkgDash := range pkg.dashboards() { + if acts.skipResource(KindDashboard, pkgDash.PkgName()) { + continue + } state.mDashboards[pkgDash.PkgName()] = &stateDashboard{ parserDash: pkgDash, stateStatus: StateStatusNew, } } for _, pkgEndpoint := range pkg.notificationEndpoints() { + if acts.skipResource(KindNotificationEndpoint, pkgEndpoint.PkgName()) { + continue + } state.mEndpoints[pkgEndpoint.PkgName()] = &stateEndpoint{ parserEndpoint: pkgEndpoint, stateStatus: StateStatusNew, } } for _, pkgLabel := range pkg.labels() { + if acts.skipResource(KindLabel, pkgLabel.PkgName()) { + continue + } state.mLabels[pkgLabel.PkgName()] = &stateLabel{ parserLabel: pkgLabel, stateStatus: StateStatusNew, } } for _, pkgRule := range pkg.notificationRules() { + if acts.skipResource(KindNotificationRule, pkgRule.PkgName()) { + continue + } state.mRules[pkgRule.PkgName()] = &stateRule{ parserRule: pkgRule, stateStatus: StateStatusNew, } } for _, pkgTask := range pkg.tasks() { + if acts.skipResource(KindTask, pkgTask.PkgName()) { + continue + } state.mTasks[pkgTask.PkgName()] = &stateTask{ parserTask: pkgTask, stateStatus: StateStatusNew, } } for _, pkgTele := range pkg.telegrafs() { + if acts.skipResource(KindTelegraf, pkgTele.PkgName()) { + continue + } state.mTelegrafs[pkgTele.PkgName()] = &stateTelegraf{ parserTelegraf: pkgTele, stateStatus: StateStatusNew, } } for _, pkgVar := range pkg.variables() { + if acts.skipResource(KindVariable, pkgVar.PkgName()) { + continue + } state.mVariables[pkgVar.PkgName()] = &stateVariable{ parserVar: pkgVar, stateStatus: StateStatusNew, @@ -1480,3 +1507,16 @@ func IsExisting(status StateStatus) bool { func IsRemoval(status StateStatus) bool { return status == StateStatusRemove } + +type resourceActions struct { + skipKinds map[Kind]bool + skipResources map[ActionSkipResource]bool +} + +func (r resourceActions) skipResource(k Kind, metaName string) bool { + key := ActionSkipResource{ + Kind: k, + MetaName: metaName, + } + return r.skipResources[key] || r.skipKinds[k] +} diff --git a/pkger/service_test.go b/pkger/service_test.go index 51a68708b81..88a90955bd5 100644 --- a/pkger/service_test.go +++ b/pkger/service_test.go @@ -38,6 +38,9 @@ func TestService(t *testing.T) { createFn: func(ctx context.Context, stack Stack) error { return nil }, + deleteFn: func(ctx context.Context, id influxdb.ID) error { + return nil + }, readFn: func(ctx context.Context, id influxdb.ID) (Stack, error) { return Stack{ID: id}, nil }, @@ -78,6 +81,69 @@ func TestService(t *testing.T) { } t.Run("DryRun", func(t *testing.T) { + type dryRunTestFields struct { + path string + kinds []Kind + skipResources []ActionSkipResource + assertFn func(*testing.T, PkgImpactSummary) + } + + testDryRunActions := func(t *testing.T, fields dryRunTestFields) { + t.Helper() + + var skipResOpts []ApplyOptFn + for _, asr := range fields.skipResources { + skipResOpts = append(skipResOpts, ApplyWithResourceSkip(asr)) + } + + testfileRunner(t, fields.path, func(t *testing.T, pkg *Pkg) { + t.Helper() + + tests := []struct { + name string + applyOpts []ApplyOptFn + }{ + { + name: "skip resources", + applyOpts: skipResOpts, + }, + } + + for _, k := range fields.kinds { + tests = append(tests, struct { + name string + applyOpts []ApplyOptFn + }{ + name: "skip kind " + k.String(), + applyOpts: []ApplyOptFn{ + ApplyWithKindSkip(ActionSkipKind{ + Kind: k, + }), + }, + }) + } + + for _, tt := range tests { + fn := func(t *testing.T) { + t.Helper() + + svc := newTestService() + + impact, err := svc.DryRun( + context.TODO(), + influxdb.ID(100), + 0, + append(tt.applyOpts, ApplyWithPkg(pkg))..., + ) + require.NoError(t, err) + + fields.assertFn(t, impact) + } + t.Run(tt.name, fn) + } + }) + } + t.Run("buckets", func(t *testing.T) { t.Run("single bucket updated", func(t *testing.T) { testfileRunner(t, "testdata/bucket.yml", func(t *testing.T, pkg *Pkg) { @@ -150,45 +216,109 @@ func TestService(t *testing.T) { assert.Contains(t, impact.Diff.Buckets, expected) }) }) + + t.Run("with actions applied", func(t *testing.T) { + testDryRunActions(t, dryRunTestFields{ + path: "testdata/bucket.yml", + kinds: []Kind{KindBucket}, + skipResources: []ActionSkipResource{ + { + Kind: KindBucket, + MetaName: "rucket-22", + }, + { + Kind: KindBucket, + MetaName: "rucket-11", + }, + }, + assertFn: func(t *testing.T, impact PkgImpactSummary) { + require.Empty(t, impact.Diff.Buckets) + }, + }) + }) }) t.Run("checks", func(t *testing.T) { - testfileRunner(t, "testdata/checks.yml", func(t *testing.T, pkg *Pkg) { - fakeCheckSVC := mock.NewCheckService() - id := influxdb.ID(1) - existing := &icheck.Deadman{ - Base: icheck.Base{ - ID: id, - Name: "display name", - Description: "old desc", - }, - } - fakeCheckSVC.FindCheckFn = func(ctx context.Context, f influxdb.CheckFilter) (influxdb.Check, error) { - if f.Name != nil && *f.Name == "display name" { - return existing, nil + t.Run("mixed update and creates", func(t *testing.T) { + testfileRunner(t, "testdata/checks.yml", func(t *testing.T, pkg *Pkg) { + fakeCheckSVC := mock.NewCheckService() + id := influxdb.ID(1) + existing := &icheck.Deadman{ + Base: icheck.Base{ + ID: id, + Name: "display name", + Description: "old desc", + }, + } + fakeCheckSVC.FindCheckFn = func(ctx context.Context, f influxdb.CheckFilter) (influxdb.Check, error) { + if f.Name != nil && *f.Name == "display name" { + return existing, nil + } + return nil, errors.New("not found") } - return nil, errors.New("not found") - } - svc := newTestService(WithCheckSVC(fakeCheckSVC)) + svc := newTestService(WithCheckSVC(fakeCheckSVC)) - impact, err := svc.DryRun(context.TODO(), influxdb.ID(100), 0, ApplyWithPkg(pkg)) - require.NoError(t, err) + impact, err := svc.DryRun(context.TODO(), influxdb.ID(100), 0, ApplyWithPkg(pkg)) + require.NoError(t, err) - checks := impact.Diff.Checks - require.Len(t, checks, 2) - check0 := checks[0] - assert.True(t, check0.IsNew()) - assert.Equal(t, "check-0", check0.PkgName) - assert.Zero(t, check0.ID) - assert.Nil(t, check0.Old) - - check1 := checks[1] - assert.False(t, check1.IsNew()) - assert.Equal(t, "check-1", check1.PkgName) - assert.Equal(t, "display name", check1.New.GetName()) - assert.NotZero(t, check1.ID) - assert.Equal(t, existing, check1.Old.Check) + checks := impact.Diff.Checks + require.Len(t, checks, 2) + check0 := checks[0] + assert.True(t, check0.IsNew()) + assert.Equal(t, "check-0", check0.PkgName) + assert.Zero(t, check0.ID) + assert.Nil(t, check0.Old) + + check1 := checks[1] + assert.False(t, check1.IsNew()) + assert.Equal(t, "check-1", check1.PkgName) + assert.Equal(t, "display name", check1.New.GetName()) + assert.NotZero(t, check1.ID) + assert.Equal(t, existing, check1.Old.Check) + }) + }) + + t.Run("with actions applied", func(t *testing.T) { + testDryRunActions(t, dryRunTestFields{ + path: "testdata/checks.yml", + kinds: []Kind{KindCheck, KindCheckDeadman, KindCheckThreshold}, + skipResources: []ActionSkipResource{ + { + Kind: KindCheck, + MetaName: "check-0", + }, + { + Kind: KindCheck, + MetaName: "check-1", + }, + }, + assertFn: func(t *testing.T, impact PkgImpactSummary) { + require.Empty(t, impact.Diff.Checks) + }, + }) + }) + }) + + t.Run("dashboards", func(t *testing.T) { + t.Run("with actions applied", func(t *testing.T) { + testDryRunActions(t, dryRunTestFields{ + path: "testdata/dashboard.yml", + kinds: []Kind{KindDashboard}, + skipResources: []ActionSkipResource{ + { + Kind: KindDashboard, + MetaName: "dash-1", + }, + { + Kind: KindDashboard, + MetaName: "dash-2", + }, + }, + assertFn: func(t *testing.T, impact PkgImpactSummary) { + require.Empty(t, impact.Diff.Dashboards) + }, + }) }) }) @@ -277,122 +407,203 @@ func TestService(t *testing.T) { assert.Contains(t, labels, expected) }) }) + + t.Run("with actions applied", func(t *testing.T) { + testDryRunActions(t, dryRunTestFields{ + path: "testdata/label.yml", + kinds: []Kind{KindLabel}, + skipResources: []ActionSkipResource{ + { + Kind: KindLabel, + MetaName: "label-1", + }, + { + Kind: KindLabel, + MetaName: "label-2", + }, + { + Kind: KindLabel, + MetaName: "label-3", + }, + }, + assertFn: func(t *testing.T, impact PkgImpactSummary) { + require.Empty(t, impact.Diff.Labels) + }, + }) + }) }) t.Run("notification endpoints", func(t *testing.T) { - testfileRunner(t, "testdata/notification_endpoint.yml", func(t *testing.T, pkg *Pkg) { - fakeEndpointSVC := mock.NewNotificationEndpointService() - id := influxdb.ID(1) - existing := &endpoint.HTTP{ - Base: endpoint.Base{ - ID: &id, - Name: "http-none-auth-notification-endpoint", - Description: "old desc", - Status: influxdb.TaskStatusInactive, - }, - Method: "POST", - AuthMethod: "none", - URL: "https://www.example.com/endpoint/old", - } - fakeEndpointSVC.FindNotificationEndpointsF = func(ctx context.Context, f influxdb.NotificationEndpointFilter, opt ...influxdb.FindOptions) ([]influxdb.NotificationEndpoint, int, error) { - return []influxdb.NotificationEndpoint{existing}, 1, nil - } + t.Run("mixed update and created", func(t *testing.T) { + testfileRunner(t, "testdata/notification_endpoint.yml", func(t *testing.T, pkg *Pkg) { + fakeEndpointSVC := mock.NewNotificationEndpointService() + id := influxdb.ID(1) + existing := &endpoint.HTTP{ + Base: endpoint.Base{ + ID: &id, + Name: "http-none-auth-notification-endpoint", + Description: "old desc", + Status: influxdb.TaskStatusInactive, + }, + Method: "POST", + AuthMethod: "none", + URL: "https://www.example.com/endpoint/old", + } + fakeEndpointSVC.FindNotificationEndpointsF = func(ctx context.Context, f influxdb.NotificationEndpointFilter, opt ...influxdb.FindOptions) ([]influxdb.NotificationEndpoint, int, error) { + return []influxdb.NotificationEndpoint{existing}, 1, nil + } - svc := newTestService(WithNotificationEndpointSVC(fakeEndpointSVC)) + svc := newTestService(WithNotificationEndpointSVC(fakeEndpointSVC)) - impact, err := svc.DryRun(context.TODO(), influxdb.ID(100), 0, ApplyWithPkg(pkg)) - require.NoError(t, err) + impact, err := svc.DryRun(context.TODO(), influxdb.ID(100), 0, ApplyWithPkg(pkg)) + require.NoError(t, err) - require.Len(t, impact.Diff.NotificationEndpoints, 5) + require.Len(t, impact.Diff.NotificationEndpoints, 5) - var ( - newEndpoints []DiffNotificationEndpoint - existingEndpoints []DiffNotificationEndpoint - ) - for _, e := range impact.Diff.NotificationEndpoints { - if e.Old != nil { - existingEndpoints = append(existingEndpoints, e) - continue + var ( + newEndpoints []DiffNotificationEndpoint + existingEndpoints []DiffNotificationEndpoint + ) + for _, e := range impact.Diff.NotificationEndpoints { + if e.Old != nil { + existingEndpoints = append(existingEndpoints, e) + continue + } + newEndpoints = append(newEndpoints, e) } - newEndpoints = append(newEndpoints, e) - } - require.Len(t, newEndpoints, 4) - require.Len(t, existingEndpoints, 1) - - expected := DiffNotificationEndpoint{ - DiffIdentifier: DiffIdentifier{ - ID: 1, - PkgName: "http-none-auth-notification-endpoint", - StateStatus: StateStatusExists, - }, - Old: &DiffNotificationEndpointValues{ - NotificationEndpoint: existing, - }, - New: DiffNotificationEndpointValues{ - NotificationEndpoint: &endpoint.HTTP{ - Base: endpoint.Base{ - ID: &id, - Name: "http-none-auth-notification-endpoint", - Description: "http none auth desc", - Status: influxdb.TaskStatusActive, + require.Len(t, newEndpoints, 4) + require.Len(t, existingEndpoints, 1) + + expected := DiffNotificationEndpoint{ + DiffIdentifier: DiffIdentifier{ + ID: 1, + PkgName: "http-none-auth-notification-endpoint", + StateStatus: StateStatusExists, + }, + Old: &DiffNotificationEndpointValues{ + NotificationEndpoint: existing, + }, + New: DiffNotificationEndpointValues{ + NotificationEndpoint: &endpoint.HTTP{ + Base: endpoint.Base{ + ID: &id, + Name: "http-none-auth-notification-endpoint", + Description: "http none auth desc", + Status: influxdb.TaskStatusActive, + }, + AuthMethod: "none", + Method: "GET", + URL: "https://www.example.com/endpoint/noneauth", }, - AuthMethod: "none", - Method: "GET", - URL: "https://www.example.com/endpoint/noneauth", }, + } + assert.Equal(t, expected, existingEndpoints[0]) + }) + }) + + t.Run("with actions applied", func(t *testing.T) { + testDryRunActions(t, dryRunTestFields{ + path: "testdata/notification_endpoint.yml", + kinds: []Kind{ + KindNotificationEndpoint, + KindNotificationEndpointHTTP, + KindNotificationEndpointPagerDuty, + KindNotificationEndpointSlack, }, - } - assert.Equal(t, expected, existingEndpoints[0]) + skipResources: []ActionSkipResource{ + { + Kind: KindNotificationEndpoint, + MetaName: "http-none-auth-notification-endpoint", + }, + { + Kind: KindNotificationEndpoint, + MetaName: "http-bearer-auth-notification-endpoint", + }, + { + Kind: KindNotificationEndpointHTTP, + MetaName: "http-basic-auth-notification-endpoint", + }, + { + Kind: KindNotificationEndpointSlack, + MetaName: "slack-notification-endpoint", + }, + { + Kind: KindNotificationEndpointPagerDuty, + MetaName: "pager-duty-notification-endpoint", + }, + }, + assertFn: func(t *testing.T, impact PkgImpactSummary) { + require.Empty(t, impact.Diff.NotificationEndpoints) + }, + }) }) }) t.Run("notification rules", func(t *testing.T) { - testfileRunner(t, "testdata/notification_rule.yml", func(t *testing.T, pkg *Pkg) { - fakeEndpointSVC := mock.NewNotificationEndpointService() - id := influxdb.ID(1) - existing := &endpoint.HTTP{ - Base: endpoint.Base{ - ID: &id, - // This name here matches the endpoint identified in the pkg notification rule - Name: "endpoint-0", - Description: "old desc", - Status: influxdb.TaskStatusInactive, - }, - Method: "POST", - AuthMethod: "none", - URL: "https://www.example.com/endpoint/old", - } - fakeEndpointSVC.FindNotificationEndpointsF = func(ctx context.Context, f influxdb.NotificationEndpointFilter, opt ...influxdb.FindOptions) ([]influxdb.NotificationEndpoint, int, error) { - return []influxdb.NotificationEndpoint{existing}, 1, nil - } + t.Run("mixed update and created", func(t *testing.T) { + testfileRunner(t, "testdata/notification_rule.yml", func(t *testing.T, pkg *Pkg) { + fakeEndpointSVC := mock.NewNotificationEndpointService() + id := influxdb.ID(1) + existing := &endpoint.HTTP{ + Base: endpoint.Base{ + ID: &id, + // This name here matches the endpoint identified in the pkg notification rule + Name: "endpoint-0", + Description: "old desc", + Status: influxdb.TaskStatusInactive, + }, + Method: "POST", + AuthMethod: "none", + URL: "https://www.example.com/endpoint/old", + } + fakeEndpointSVC.FindNotificationEndpointsF = func(ctx context.Context, f influxdb.NotificationEndpointFilter, opt ...influxdb.FindOptions) ([]influxdb.NotificationEndpoint, int, error) { + return []influxdb.NotificationEndpoint{existing}, 1, nil + } - svc := newTestService(WithNotificationEndpointSVC(fakeEndpointSVC)) + svc := newTestService(WithNotificationEndpointSVC(fakeEndpointSVC)) - impact, err := svc.DryRun(context.TODO(), influxdb.ID(100), 0, ApplyWithPkg(pkg)) - require.NoError(t, err) + impact, err := svc.DryRun(context.TODO(), influxdb.ID(100), 0, ApplyWithPkg(pkg)) + require.NoError(t, err) - require.Len(t, impact.Diff.NotificationRules, 1) + require.Len(t, impact.Diff.NotificationRules, 1) - actual := impact.Diff.NotificationRules[0].New - assert.Equal(t, "rule_0", actual.Name) - assert.Equal(t, "desc_0", actual.Description) - assert.Equal(t, "slack", actual.EndpointType) - assert.Equal(t, existing.Name, actual.EndpointName) - assert.Equal(t, SafeID(*existing.ID), actual.EndpointID) - assert.Equal(t, (10 * time.Minute).String(), actual.Every) - assert.Equal(t, (30 * time.Second).String(), actual.Offset) + actual := impact.Diff.NotificationRules[0].New + assert.Equal(t, "rule_0", actual.Name) + assert.Equal(t, "desc_0", actual.Description) + assert.Equal(t, "slack", actual.EndpointType) + assert.Equal(t, existing.Name, actual.EndpointName) + assert.Equal(t, SafeID(*existing.ID), actual.EndpointID) + assert.Equal(t, (10 * time.Minute).String(), actual.Every) + assert.Equal(t, (30 * time.Second).String(), actual.Offset) - expectedStatusRules := []SummaryStatusRule{ - {CurrentLevel: "CRIT", PreviousLevel: "OK"}, - {CurrentLevel: "WARN"}, - } - assert.Equal(t, expectedStatusRules, actual.StatusRules) + expectedStatusRules := []SummaryStatusRule{ + {CurrentLevel: "CRIT", PreviousLevel: "OK"}, + {CurrentLevel: "WARN"}, + } + assert.Equal(t, expectedStatusRules, actual.StatusRules) - expectedTagRules := []SummaryTagRule{ - {Key: "k1", Value: "v1", Operator: "equal"}, - {Key: "k1", Value: "v2", Operator: "equal"}, - } - assert.Equal(t, expectedTagRules, actual.TagRules) + expectedTagRules := []SummaryTagRule{ + {Key: "k1", Value: "v1", Operator: "equal"}, + {Key: "k1", Value: "v2", Operator: "equal"}, + } + assert.Equal(t, expectedTagRules, actual.TagRules) + }) + }) + + t.Run("with actions applied", func(t *testing.T) { + testDryRunActions(t, dryRunTestFields{ + path: "testdata/notification_rule.yml", + kinds: []Kind{KindNotificationRule}, + skipResources: []ActionSkipResource{ + { + Kind: KindNotificationRule, + MetaName: "rule-uuid", + }, + }, + assertFn: func(t *testing.T, impact PkgImpactSummary) { + require.Empty(t, impact.Diff.NotificationRules) + }, + }) }) }) @@ -411,63 +622,137 @@ func TestService(t *testing.T) { }) }) - t.Run("variables", func(t *testing.T) { - testfileRunner(t, "testdata/variables.json", func(t *testing.T, pkg *Pkg) { - fakeVarSVC := mock.NewVariableService() - fakeVarSVC.FindVariablesF = func(_ context.Context, filter influxdb.VariableFilter, opts ...influxdb.FindOptions) ([]*influxdb.Variable, error) { - return []*influxdb.Variable{ + t.Run("tasks", func(t *testing.T) { + t.Run("with actions applied", func(t *testing.T) { + testDryRunActions(t, dryRunTestFields{ + path: "testdata/tasks.yml", + kinds: []Kind{KindTask}, + skipResources: []ActionSkipResource{ { - ID: influxdb.ID(1), - Name: "var-const-3", - Description: "old desc", + Kind: KindTask, + MetaName: "task-uuid", + }, + { + Kind: KindTask, + MetaName: "task-1", }, - }, nil - } - svc := newTestService(WithVariableSVC(fakeVarSVC)) - - impact, err := svc.DryRun(context.TODO(), influxdb.ID(100), 0, ApplyWithPkg(pkg)) - require.NoError(t, err) - - variables := impact.Diff.Variables - require.Len(t, variables, 4) - - expected := DiffVariable{ - DiffIdentifier: DiffIdentifier{ - ID: 1, - PkgName: "var-const-3", - StateStatus: StateStatusExists, }, - Old: &DiffVariableValues{ - Name: "var-const-3", - Description: "old desc", + assertFn: func(t *testing.T, impact PkgImpactSummary) { + require.Empty(t, impact.Diff.Tasks) }, - New: DiffVariableValues{ - Name: "var-const-3", - Description: "var-const-3 desc", - Args: &influxdb.VariableArguments{ - Type: "constant", - Values: influxdb.VariableConstantValues{"first val"}, + }) + }) + }) + + t.Run("telegraf configs", func(t *testing.T) { + t.Run("with actions applied", func(t *testing.T) { + testDryRunActions(t, dryRunTestFields{ + path: "testdata/telegraf.yml", + kinds: []Kind{KindTelegraf}, + skipResources: []ActionSkipResource{ + { + Kind: KindTelegraf, + MetaName: "first-tele-config", + }, + { + Kind: KindTelegraf, + MetaName: "tele-2", }, }, - } - assert.Equal(t, expected, variables[0]) - - expected = DiffVariable{ - DiffIdentifier: DiffIdentifier{ - // no ID here since this one would be new - PkgName: "var-map-4", - StateStatus: StateStatusNew, + assertFn: func(t *testing.T, impact PkgImpactSummary) { + require.Empty(t, impact.Diff.Telegrafs) }, - New: DiffVariableValues{ - Name: "var-map-4", - Description: "var-map-4 desc", - Args: &influxdb.VariableArguments{ - Type: "map", - Values: influxdb.VariableMapValues{"k1": "v1"}, + }) + }) + }) + + t.Run("variables", func(t *testing.T) { + t.Run("mixed update and created", func(t *testing.T) { + testfileRunner(t, "testdata/variables.json", func(t *testing.T, pkg *Pkg) { + fakeVarSVC := mock.NewVariableService() + fakeVarSVC.FindVariablesF = func(_ context.Context, filter influxdb.VariableFilter, opts ...influxdb.FindOptions) ([]*influxdb.Variable, error) { + return []*influxdb.Variable{ + { + ID: influxdb.ID(1), + Name: "var-const-3", + Description: "old desc", + }, + }, nil + } + svc := newTestService(WithVariableSVC(fakeVarSVC)) + + impact, err := svc.DryRun(context.TODO(), influxdb.ID(100), 0, ApplyWithPkg(pkg)) + require.NoError(t, err) + + variables := impact.Diff.Variables + require.Len(t, variables, 4) + + expected := DiffVariable{ + DiffIdentifier: DiffIdentifier{ + ID: 1, + PkgName: "var-const-3", + StateStatus: StateStatusExists, + }, + Old: &DiffVariableValues{ + Name: "var-const-3", + Description: "old desc", + }, + New: DiffVariableValues{ + Name: "var-const-3", + Description: "var-const-3 desc", + Args: &influxdb.VariableArguments{ + Type: "constant", + Values: influxdb.VariableConstantValues{"first val"}, + }, + }, + } + assert.Equal(t, expected, variables[0]) + + expected = DiffVariable{ + DiffIdentifier: DiffIdentifier{ + // no ID here since this one would be new + PkgName: "var-map-4", + StateStatus: StateStatusNew, + }, + New: DiffVariableValues{ + Name: "var-map-4", + Description: "var-map-4 desc", + Args: &influxdb.VariableArguments{ + Type: "map", + Values: influxdb.VariableMapValues{"k1": "v1"}, + }, + }, + } + assert.Equal(t, expected, variables[1]) + }) + }) + + t.Run("with actions applied", func(t *testing.T) { + testDryRunActions(t, dryRunTestFields{ + path: "testdata/variables.yml", + kinds: []Kind{KindVariable}, + skipResources: []ActionSkipResource{ + { + Kind: KindVariable, + MetaName: "var-query-1", + }, + { + Kind: KindVariable, + MetaName: "var-query-2", + }, + { + Kind: KindVariable, + MetaName: "var-const-3", + }, + { + Kind: KindVariable, + MetaName: "var-map-4", }, }, - } - assert.Equal(t, expected, variables[1]) + assertFn: func(t *testing.T, impact PkgImpactSummary) { + require.Empty(t, impact.Diff.Variables) + }, + }) }) }) }) @@ -3372,6 +3657,7 @@ func levelPtr(l notification.CheckLevel) *notification.CheckLevel { type fakeStore struct { createFn func(ctx context.Context, stack Stack) error + deleteFn func(ctx context.Context, id influxdb.ID) error readFn func(ctx context.Context, id influxdb.ID) (Stack, error) updateFn func(ctx context.Context, stack Stack) error } @@ -3404,6 +3690,9 @@ func (s *fakeStore) UpdateStack(ctx context.Context, stack Stack) error { } func (s *fakeStore) DeleteStack(ctx context.Context, id influxdb.ID) error { + if s.deleteFn != nil { + return s.deleteFn(ctx, id) + } panic("not implemented") } diff --git a/tsdb/tsm1/engine_schema.go b/tsdb/tsm1/engine_schema.go index defe2a46c77..87fa2061df8 100644 --- a/tsdb/tsm1/engine_schema.go +++ b/tsdb/tsm1/engine_schema.go @@ -358,6 +358,9 @@ func (e *Engine) tagKeysNoPredicate(ctx context.Context, orgID, bucketID influxd var stats cursors.CursorStats var canceled bool + var files unrefs + defer func() { files.Unref() }() + e.FileStore.ForEachFile(func(f TSMFile) bool { // Check the context before touching each tsm file select { @@ -366,6 +369,8 @@ func (e *Engine) tagKeysNoPredicate(ctx context.Context, orgID, bucketID influxd return false default: } + + var hasRef bool if f.OverlapsTimeRange(start, end) && f.OverlapsKeyPrefixRange(tsmKeyPrefix, tsmKeyPrefix) { // TODO(sgc): create f.TimeRangeIterator(minKey, maxKey, start, end) iter := f.TimeRangeIterator(tsmKeyPrefix, start, end) @@ -384,6 +389,12 @@ func (e *Engine) tagKeysNoPredicate(ctx context.Context, orgID, bucketID influxd if iter.HasData() { keyset.UnionKeys(tags) + + // Add reference to ensure tags are valid for the outer function. + if !hasRef { + f.Ref() + files, hasRef = append(files, f), true + } } } stats.Add(iter.Stats()) diff --git a/ui/src/cloud/apis/reporting.ts b/ui/src/cloud/apis/reporting.ts index 3866c808428..01a8faf7df5 100644 --- a/ui/src/cloud/apis/reporting.ts +++ b/ui/src/cloud/apis/reporting.ts @@ -21,7 +21,7 @@ export interface Points { export const reportPoints = (points: Points) => { try { const url = '/api/v2/app-metrics' - fetch(url, { + return fetch(url, { method: 'POST', body: JSON.stringify(points), headers: { diff --git a/ui/src/cloud/utils/rum.ts b/ui/src/cloud/utils/rum.ts index 3e7cf5b0d4a..313ff60edcd 100644 --- a/ui/src/cloud/utils/rum.ts +++ b/ui/src/cloud/utils/rum.ts @@ -3,6 +3,7 @@ import {getFlags} from 'src/client' import {reportError} from 'src/shared/utils/errors' +import {reportPoints} from 'src/cloud/apis/reporting' export interface Tags { [key: string]: string @@ -107,13 +108,6 @@ export const writeNavigationTimingMetrics = async function writeNavigationTiming const points = {points: [{measurement, tags, fields}]} - fetch('/api/v2/app-metrics', { - method: 'POST', - body: JSON.stringify(points), - headers: { - 'Content-Type': 'application/json', - }, - credentials: 'include', - }) + reportPoints(points) } } diff --git a/ui/src/organizations/components/OrgProfileTab.tsx b/ui/src/organizations/components/OrgProfileTab.tsx index e89e99ce5bc..b5dc2505e9c 100644 --- a/ui/src/organizations/components/OrgProfileTab.tsx +++ b/ui/src/organizations/components/OrgProfileTab.tsx @@ -1,5 +1,6 @@ // Libraries import React, {PureComponent} from 'react' +import {connect} from 'react-redux' import {WithRouterProps, withRouter} from 'react-router' import _ from 'lodash' @@ -17,53 +18,92 @@ import { Gradients, InfluxColors, JustifyContent, + Grid, + Columns, } from '@influxdata/clockface' import {ErrorHandling} from 'src/shared/decorators/errors' +import CodeSnippet from 'src/shared/components/CodeSnippet' + +import {getOrg} from 'src/organizations/selectors' +import { + copyToClipboardSuccess, + copyToClipboardFailed, +} from 'src/shared/copy/notifications' // Types import {ButtonType} from 'src/clockface' +import {AppState, Organization} from 'src/types' +import {MeState} from 'src/shared/reducers/me' -type Props = WithRouterProps +interface StateProps { + me: MeState + org: Organization +} + +type Props = StateProps & WithRouterProps @ErrorHandling class OrgProfileTab extends PureComponent { public render() { return ( - - -

Organization Profile

- - -
- - -
Danger Zone!
- - - -
-
Rename Organization
-

- This action can have wide-reaching unintended - consequences. -

-
-