From 023d43d0fe4e2a1b4f0fbb21501fecc422a503db Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Fri, 14 Jun 2019 15:11:15 -0400 Subject: [PATCH 001/118] Add make serve-docs target for local viewing Signed-off-by: Nolan Brubaker --- Makefile | 8 ++++++++ site/README-JEKYLL.md | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/Makefile b/Makefile index 288cca4fcb8..31efb63f8c9 100644 --- a/Makefile +++ b/Makefile @@ -214,3 +214,11 @@ changelog: release: hack/goreleaser.sh + +serve-docs: + docker run \ + --rm \ + -v "$$(pwd)/site:/srv/jekyll" \ + -it -p 4000:4000 \ + jekyll/jekyll \ + jekyll serve --livereload diff --git a/site/README-JEKYLL.md b/site/README-JEKYLL.md index 9d5d2205cbf..5ed0b28a6c5 100644 --- a/site/README-JEKYLL.md +++ b/site/README-JEKYLL.md @@ -1,3 +1,7 @@ +# Running in Docker + +To run this site in a Docker container, you can use `make serve-docs` from the root directory. + # Dependencies for MacOS Install the following for an easy to use dev environment: From 5d3312b7b5c6d4eadfa61b84fe7d6a3adec7deef Mon Sep 17 00:00:00 2001 From: Ross Fenrick Date: Tue, 18 Jun 2019 13:14:13 -0500 Subject: [PATCH 002/118] added top spacing and reduced bottom (#1583) * added top spacing and reduced bottom Signed-off-by: Ross Fenrick * added marign to h3's Signed-off-by: Ross Fenrick --- site/_scss/site/common/_type.scss | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/site/_scss/site/common/_type.scss b/site/_scss/site/common/_type.scss index 8ea3f159cf2..d6e653db180 100644 --- a/site/_scss/site/common/_type.scss +++ b/site/_scss/site/common/_type.scss @@ -25,7 +25,12 @@ h1 { h2 { color: $body-color-darkest; - margin-bottom: 2rem; + margin-bottom: 1rem; + margin-top: 2rem; +} + +h3 { + margin-top: 2rem; } h5 { From 2d81e29276283f3819298758f1381916b0f0c42e Mon Sep 17 00:00:00 2001 From: Adnan Abdulhussein Date: Tue, 18 Jun 2019 11:19:00 -0700 Subject: [PATCH 003/118] ensure backup item action modifications reflected in tarball filepath (#1587) * ensure backup item action modifications reflected in tarball filepath This patch ensures the updated backup item's name and namespace are used when constructing the filepath for the tarball. Signed-off-by: Adnan Abdulhussein * changelog Signed-off-by: Adnan Abdulhussein --- changelogs/unreleased/1587-prydonius | 1 + pkg/backup/backup_new_test.go | 5 ++--- pkg/backup/item_backupper.go | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changelogs/unreleased/1587-prydonius diff --git a/changelogs/unreleased/1587-prydonius b/changelogs/unreleased/1587-prydonius new file mode 100644 index 00000000000..db3cc1ced4d --- /dev/null +++ b/changelogs/unreleased/1587-prydonius @@ -0,0 +1 @@ +ensures backup item action modifications to an item's namespace/name are saved in the file path in the tarball diff --git a/pkg/backup/backup_new_test.go b/pkg/backup/backup_new_test.go index 639622e1521..7a8b9333881 100644 --- a/pkg/backup/backup_new_test.go +++ b/pkg/backup/backup_new_test.go @@ -1006,8 +1006,7 @@ func TestBackupActionModifications(t *testing.T) { }, }, { - // TODO this seems like a bug - name: "modifications to name and namespace in an action are persisted in JSON but not in filename", + name: "modifications to name and namespace in an action are persisted in JSON and in filename", backup: defaultBackup(). Backup(), apiResources: []*apiResource{ @@ -1022,7 +1021,7 @@ func TestBackupActionModifications(t *testing.T) { }), }, want: map[string]unstructuredObject{ - "resources/pods/namespaces/ns-1/pod-1.json": toUnstructuredOrFail(t, newPod("ns-1-updated", "pod-1-updated")), + "resources/pods/namespaces/ns-1-updated/pod-1-updated.json": toUnstructuredOrFail(t, newPod("ns-1-updated", "pod-1-updated")), }, }, } diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 6b4b8369812..189f562c030 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -206,6 +206,9 @@ func (ib *defaultItemBackupper) backupItem(logger logrus.FieldLogger, obj runtim if metadata, err = meta.Accessor(obj); err != nil { return errors.WithStack(err) } + // update name and namespace in case they were modified in an action + name = metadata.GetName() + namespace = metadata.GetNamespace() if groupResource == kuberesource.PersistentVolumes { if err := ib.takePVSnapshot(obj, log); err != nil { From 1f4139a5bfc204d3aca127ba90fb8868a1814e3a Mon Sep 17 00:00:00 2001 From: Adnan Abdulhussein Date: Wed, 19 Jun 2019 07:58:02 -0700 Subject: [PATCH 004/118] allow exclusion of resources using standard label (#1588) * allow exclusion of resources using standard label excludes any resources with the velero.io/exclude-from-backup=true label Signed-off-by: Adnan Abdulhussein --- changelogs/unreleased/1588-prydonius | 1 + pkg/backup/backup_new_test.go | 84 +++++++++++++++++++++++++++- pkg/backup/resource_backupper.go | 4 +- 3 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 changelogs/unreleased/1588-prydonius diff --git a/changelogs/unreleased/1588-prydonius b/changelogs/unreleased/1588-prydonius new file mode 100644 index 00000000000..a8481eff446 --- /dev/null +++ b/changelogs/unreleased/1588-prydonius @@ -0,0 +1 @@ +allows excluding resources from backups with the velero.io/exclude-from-backup=true label diff --git a/pkg/backup/backup_new_test.go b/pkg/backup/backup_new_test.go index 7a8b9333881..ca5912910ec 100644 --- a/pkg/backup/backup_new_test.go +++ b/pkg/backup/backup_new_test.go @@ -218,6 +218,81 @@ func TestBackupResourceFiltering(t *testing.T) { "resources/persistentvolumes/cluster/bar.json", }, }, + { + name: "resources with velero.io/exclude-from-backup=true label are not included", + backup: defaultBackup(). + Backup(), + apiResources: []*apiResource{ + pods( + withLabel(newPod("foo", "bar"), "velero.io/exclude-from-backup", "true"), + newPod("zoo", "raz"), + ), + deployments( + newDeployment("foo", "bar"), + withLabel(newDeployment("zoo", "raz"), "velero.io/exclude-from-backup", "true"), + ), + pvs( + withLabel(newPV("bar"), "a", "b"), + withLabel(newPV("baz"), "velero.io/exclude-from-backup", "true"), + ), + }, + want: []string{ + "resources/pods/namespaces/zoo/raz.json", + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/persistentvolumes/cluster/bar.json", + }, + }, + { + name: "resources with velero.io/exclude-from-backup=true label are not included even if matching label selector", + backup: defaultBackup(). + LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). + Backup(), + apiResources: []*apiResource{ + pods( + withLabel(newPod("foo", "bar"), "velero.io/exclude-from-backup", "true", "a", "b"), + withLabel(newPod("zoo", "raz"), "a", "b"), + ), + deployments( + newDeployment("foo", "bar"), + withLabel(newDeployment("zoo", "raz"), "velero.io/exclude-from-backup", "true", "a", "b"), + ), + pvs( + withLabel(newPV("bar"), "a", "b"), + withLabel(newPV("baz"), "a", "b", "velero.io/exclude-from-backup", "true"), + ), + }, + want: []string{ + "resources/pods/namespaces/zoo/raz.json", + "resources/persistentvolumes/cluster/bar.json", + }, + }, + { + name: "resources with velero.io/exclude-from-backup label specified but not 'true' are included", + backup: defaultBackup(). + Backup(), + apiResources: []*apiResource{ + pods( + withLabel(newPod("foo", "bar"), "velero.io/exclude-from-backup", "false"), + newPod("zoo", "raz"), + ), + deployments( + newDeployment("foo", "bar"), + withLabel(newDeployment("zoo", "raz"), "velero.io/exclude-from-backup", "1"), + ), + pvs( + withLabel(newPV("bar"), "a", "b"), + withLabel(newPV("baz"), "velero.io/exclude-from-backup", ""), + ), + }, + want: []string{ + "resources/pods/namespaces/foo/bar.json", + "resources/pods/namespaces/zoo/raz.json", + "resources/deployments.apps/namespaces/foo/bar.json", + "resources/deployments.apps/namespaces/zoo/raz.json", + "resources/persistentvolumes/cluster/bar.json", + "resources/persistentvolumes/cluster/baz.json", + }, + }, { name: "should include cluster-scoped resources if backing up subset of namespaces and IncludeClusterResources=true", backup: defaultBackup(). @@ -1794,12 +1869,17 @@ func newHarness(t *testing.T) *harness { } } -func withLabel(obj metav1.Object, key, val string) metav1.Object { +func withLabel(obj metav1.Object, labelPairs ...string) metav1.Object { + if len(labelPairs)%2 != 0 { + panic("withLabel requires a series of key-value pairs") + } labels := obj.GetLabels() if labels == nil { labels = make(map[string]string) } - labels[key] = val + for i := 0; i < len(labelPairs); i += 2 { + labels[labelPairs[i]] = labelPairs[i+1] + } obj.SetLabels(labels) return obj diff --git a/pkg/backup/resource_backupper.go b/pkg/backup/resource_backupper.go index 86d1ca8a496..de78a25d390 100644 --- a/pkg/backup/resource_backupper.go +++ b/pkg/backup/resource_backupper.go @@ -221,9 +221,9 @@ func (rb *defaultResourceBackupper) backupResource(group *metav1.APIResourceList continue } - var labelSelector string + labelSelector := "velero.io/exclude-from-backup!=true" if selector := rb.backupRequest.Spec.LabelSelector; selector != nil { - labelSelector = metav1.FormatLabelSelector(selector) + labelSelector = labelSelector + "," + metav1.FormatLabelSelector(selector) } log.Info("Listing items") From 870743a28d592e4e1046d741d91936d1f37982d9 Mon Sep 17 00:00:00 2001 From: Jonas Rosland Date: Wed, 19 Jun 2019 12:20:56 -0400 Subject: [PATCH 005/118] Add redirect for docs (#1584) * Add redirects for docs Signed-off-by: jonasrosland --- README.md | 10 +- site/_config.yml | 8 +- site/_data/shortlinks.yml | 9 ++ site/_layouts/redirect.html | 13 +++ site/_plugins/data_page_generator.rb | 135 +++++++++++++++++++++++++++ site/docs/index.md | 4 + 6 files changed, 173 insertions(+), 6 deletions(-) create mode 100644 site/_data/shortlinks.yml create mode 100644 site/_layouts/redirect.html create mode 100644 site/_plugins/data_page_generator.rb create mode 100644 site/docs/index.md diff --git a/README.md b/README.md index 3d6d41503f1..61f3637fb7a 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ You can run Velero in clusters on a cloud provider or on-premises. For detailed ## Installation -We strongly recommend that you use an [official release][6] of Velero. The tarballs for each release contain the +We strongly recommend that you use an [official release][6] of Velero. The tarballs for each release contain the `velero` command-line client. Follow the instructions under the **Install** section of [our documentation][29] to get started. _The code and sample YAML files in the master branch of the Velero repository are under active development and are not guaranteed to be stable. Use them at your own risk!_ @@ -70,11 +70,11 @@ See [the list of releases][6] to find out about feature changes. [24]: https://groups.google.com/forum/#!forum/projectvelero [25]: https://kubernetes.slack.com/messages/velero -[26]: https://velero.io/docs/v1.0.0/zenhub +[26]: https://velero.io/docs/zenhub -[29]: https://velero.io/docs/v1.0.0/ -[30]: https://velero.io/docs/v1.0.0/troubleshooting +[29]: https://velero.io/docs/ +[30]: https://velero.io/docs/troubleshooting -[99]: https://velero.io/docs/v1.0.0/support-matrix +[99]: https://velero.io/docs/support-matrix [100]: /site/docs/master/img/velero.png diff --git a/site/_config.yml b/site/_config.yml index 6fd6bec8f31..f71994ce3fc 100644 --- a/site/_config.yml +++ b/site/_config.yml @@ -122,6 +122,12 @@ defaults: gh: https://github.com/heptio/velero/tree/v0.3.0 layout: "docs" +page_gen: + - data: shortlinks + template: redirect + name: key + dir: docs + collections: - contributors - casestudies @@ -158,7 +164,7 @@ plugins: - jekyll-optional-front-matter # Parse Markdown files that do not have front-matter callouts - jekyll-titles-from-headings # pull the page title from the first Markdown heading when none is specified. - jekyll-paginate # pagination object for collections (e.g. posts) - + - jekyll-redirect-from # Include these subdirectories include: diff --git a/site/_data/shortlinks.yml b/site/_data/shortlinks.yml new file mode 100644 index 00000000000..ddd84c84147 --- /dev/null +++ b/site/_data/shortlinks.yml @@ -0,0 +1,9 @@ +- title: Troubleshooting + key: troubleshooting + destination: troubleshooting +- title: Support Matrix + key: support-matrix + destination: support-matrix +- title: ZenHub + key: zenhub + destination: zenhub diff --git a/site/_layouts/redirect.html b/site/_layouts/redirect.html new file mode 100644 index 00000000000..e565888a615 --- /dev/null +++ b/site/_layouts/redirect.html @@ -0,0 +1,13 @@ + + + + + + Redirecting... + + +Redirecting to {{ page.destination }}. If it doesn't load, click here. + + diff --git a/site/_plugins/data_page_generator.rb b/site/_plugins/data_page_generator.rb new file mode 100644 index 00000000000..787ac3a2207 --- /dev/null +++ b/site/_plugins/data_page_generator.rb @@ -0,0 +1,135 @@ +# coding: utf-8 +# Generate pages from individual records in yml files +# (c) 2014-2016 Adolfo Villafiorita +# Distributed under the conditions of the MIT License + +module Jekyll + + module Sanitizer + # strip characters and whitespace to create valid filenames, also lowercase + def sanitize_filename(name) + if(name.is_a? Integer) + return name.to_s + end + return name.tr( + "ÀÁÂÃÄÅàáâãäåĀāĂ㥹ÇçĆćĈĉĊċČčÐðĎďĐđÈÉÊËèéêëĒēĔĕĖėĘęĚěĜĝĞğĠġĢģĤĥĦħÌÍÎÏìíîïĨĩĪīĬĭĮįİıĴĵĶķĸĹĺĻļĽľĿŀŁłÑñŃńŅņŇňʼnŊŋÑñÒÓÔÕÖØòóôõöøŌōŎŏŐőŔŕŖŗŘřŚśŜŝŞşŠšſŢţŤťŦŧÙÚÛÜùúûüŨũŪūŬŭŮůŰűŲųŴŵÝýÿŶŷŸŹźŻżŽž", + "AAAAAAaaaaaaAaAaAaCcCcCcCcCcDdDdDdEEEEeeeeEeEeEeEeEeGgGgGgGgHhHhIIIIiiiiIiIiIiIiIiJjKkkLlLlLlLlLlNnNnNnNnnNnNnOOOOOOooooooOoOoOoRrRrRrSsSsSsSssTtTtTtUUUUuuuuUuUuUuUuUuUuWwYyyYyYZzZzZz" +).downcase.strip.gsub(' ', '-').gsub(/[^\w.-]/, '') + end + end + + # this class is used to tell Jekyll to generate a page + class DataPage < Page + include Sanitizer + + # - site and base are copied from other plugins: to be honest, I am not sure what they do + # + # - `index_files` specifies if we want to generate named folders (true) or not (false) + # - `dir` is the default output directory + # - `data` is the data defined in `_data.yml` of the record for which we are generating a page + # - `name` is the key in `data` which determines the output filename + # - `template` is the name of the template for generating the page + # - `extension` is the extension for the generated file + def initialize(site, base, index_files, dir, data, name, template, extension) + @site = site + @base = base + + # @dir is the directory where we want to output the page + # @name is the name of the page to generate + # + # the value of these variables changes according to whether we + # want to generate named folders or not + if data[name] == nil + puts "error (datapage_gen). empty value for field '#{name}' in record #{data}" + else + filename = sanitize_filename(data[name]).to_s + + @dir = dir + (index_files ? "/" + filename + "/" : "") + @name = (index_files ? "index" : filename) + "." + extension.to_s + + self.process(@name) + self.read_yaml(File.join(base, '_layouts'), template + ".html") + self.data['title'] = data[name] + # add all the information defined in _data for the current record to the + # current page (so that we can access it with liquid tags) + self.data.merge!(data) + end + end + end + + class DataPagesGenerator < Generator + safe true + + # generate loops over _config.yml/page_gen invoking the DataPage + # constructor for each record for which we want to generate a page + + def generate(site) + # page_gen_dirs determines whether we want to generate index pages + # (name/index.html) or standard files (name.html). This information + # is passed to the DataPage constructor, which sets the @dir variable + # as required by this directive + index_files = site.config['page_gen-dirs'] == true + + # data contains the specification of the data for which we want to generate + # the pages (look at the README file for its specification) + data = site.config['page_gen'] + if data + data.each do |data_spec| + index_files_for_this_data = data_spec['index_files'] != nil ? data_spec['index_files'] : index_files + template = data_spec['template'] || data_spec['data'] + name = data_spec['name'] + dir = data_spec['dir'] || data_spec['data'] + extension = data_spec['extension'] || "html" + + if site.layouts.key? template + # records is the list of records defined in _data.yml + # for which we want to generate different pages + records = nil + data_spec['data'].split('.').each do |level| + if records.nil? + records = site.data[level] + else + records = records[level] + end + end + + # apply filtering conditions: + # - filter requires the name of a boolean field + # - filter_condition evals a ruby expression + records = records.select { |r| r[data_spec['filter']] } if data_spec['filter'] + records = records.select { |record| eval(data_spec['filter_condition']) } if data_spec['filter_condition'] + + records.each do |record| + site.pages << DataPage.new(site, site.source, index_files_for_this_data, dir, record, name, template, extension) + end + else + puts "error (datapage_gen). could not find template #{template}" if not site.layouts.key? template + end + end + end + end + end + + module DataPageLinkGenerator + include Sanitizer + + # use it like this: {{input | datapage_url: dir}} + # to generate a link to a data_page. + # + # the filter is smart enough to generate different link styles + # according to the data_page-dirs directive ... + # + # ... however, the filter is not smart enough to support different + # extensions for filenames. + # + # Thus, if you use the `extension` feature of this plugin, you + # need to generate the links by hand + def datapage_url(input, dir) + extension = Jekyll.configuration({})['page_gen-dirs'] ? '/' : '.html' + "#{dir}/#{sanitize_filename(input)}#{extension}" + end + end + +end + +Liquid::Template.register_filter(Jekyll::DataPageLinkGenerator) diff --git a/site/docs/index.md b/site/docs/index.md new file mode 100644 index 00000000000..d6f2254b97e --- /dev/null +++ b/site/docs/index.md @@ -0,0 +1,4 @@ +--- +redirect_to: +- LATEST +--- From 223aec82003dc6acbccf2410a1259c35c90d9478 Mon Sep 17 00:00:00 2001 From: jonasrosland Date: Thu, 20 Jun 2019 13:35:32 -0400 Subject: [PATCH 006/118] Add more community info - direct links to meetings and ZenHub Signed-off-by: jonasrosland --- site/community.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/site/community.md b/site/community.md index d6d3b897708..15d95960c1a 100644 --- a/site/community.md +++ b/site/community.md @@ -7,12 +7,12 @@ id: community If you’re a newcomer, check out the “[Good first issue](https://github.com/heptio/velero/issues?q=is%3Aopen+is%3Aissue+label%3A%22Good+first+issue%22)” and “[Help wanted](https://github.com/heptio/velero/issues?utf8=%E2%9C%93&q=is%3Aopen+is%3Aissue+label%3A%22Help+wanted%22+)” labels in the Velero repository. -* Follow us on Twitter at [@projectvelero](https://twitter.com/projectvelero) +You can follow the work we do, see our milestones, and our backlog on our [ZenHub board](https://app.zenhub.com/workspace/o/heptio/velero/boards?filterLogic=all&repos=99143276). +* Follow us on Twitter at [@projectvelero](https://twitter.com/projectvelero) * Join our Kubernetes Slack channel and talk to over 800 other community members: [#velero](https://kubernetes.slack.com/messages/velero) - * Join our [Google Group](https://groups.google.com/forum/#!forum/projectvelero) to get updates on the project and invites to community meetings. - -* Join the Velero community meetings every 1st and 3rd Tuesday: - +* Join the [Velero community meetings](https://github.com/heptio/velero-community): + - Every 1st Tuesday - [Zoom link](https://vmware.zoom.us/j/551441444) + - Every 3rd Tuesday - [Zoom link](https://vmware.zoom.us/j/324372812) * See previous community meetings on our [YouTube Channel](https://www.youtube.com/playlist?list=PL7bmigfV0EqQRysvqvqOtRNk4L5S7uqwM) From 0735ee7218166935726f0f1108511a6160475af0 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 20 Jun 2019 14:28:11 -0600 Subject: [PATCH 007/118] extract shared API test helpers to pkg/test Signed-off-by: Steve Kriss --- pkg/backup/backup_new_test.go | 868 ++++++++++++++-------------------- pkg/test/api_server.go | 57 +++ pkg/test/discovery_client.go | 23 +- pkg/test/resources.go | 168 +++++++ 4 files changed, 599 insertions(+), 517 deletions(-) create mode 100644 pkg/test/api_server.go create mode 100644 pkg/test/resources.go diff --git a/pkg/backup/backup_new_test.go b/pkg/backup/backup_new_test.go index ca5912910ec..55c1cfb4ae7 100644 --- a/pkg/backup/backup_new_test.go +++ b/pkg/backup/backup_new_test.go @@ -32,21 +32,15 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - discoveryfake "k8s.io/client-go/discovery/fake" - dynamicfake "k8s.io/client-go/dynamic/fake" - kubefake "k8s.io/client-go/kubernetes/fake" velerov1 "github.com/heptio/velero/pkg/apis/velero/v1" "github.com/heptio/velero/pkg/client" "github.com/heptio/velero/pkg/discovery" - "github.com/heptio/velero/pkg/generated/clientset/versioned/fake" "github.com/heptio/velero/pkg/kuberesource" "github.com/heptio/velero/pkg/plugin/velero" "github.com/heptio/velero/pkg/test" @@ -64,20 +58,20 @@ func TestBackupResourceFiltering(t *testing.T) { tests := []struct { name string backup *velerov1.Backup - apiResources []*apiResource + apiResources []*test.APIResource want []string }{ { name: "no filters backs up everything", backup: defaultBackup().Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + test.Deployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), }, want: []string{ @@ -92,14 +86,14 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). IncludedResources("pods"). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + test.Deployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), }, want: []string{ @@ -112,14 +106,14 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). ExcludedResources("deployments"). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + test.Deployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), }, want: []string{ @@ -132,14 +126,14 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). IncludedNamespaces("foo"). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + test.Deployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), }, want: []string{ @@ -152,14 +146,14 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). ExcludedNamespaces("zoo"). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + test.Deployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), }, want: []string{ @@ -172,18 +166,18 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). IncludeClusterResources(false). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + test.Deployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), - pvs( - newPV("bar"), - newPV("baz"), + test.PVs( + test.NewPV("bar"), + test.NewPV("baz"), ), }, want: []string{ @@ -198,18 +192,18 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). Backup(), - apiResources: []*apiResource{ - pods( - withLabel(newPod("foo", "bar"), "a", "b"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + withLabel(test.NewPod("foo", "bar"), "a", "b"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - withLabel(newDeployment("zoo", "raz"), "a", "b"), + test.Deployments( + test.NewDeployment("foo", "bar"), + withLabel(test.NewDeployment("zoo", "raz"), "a", "b"), ), - pvs( - withLabel(newPV("bar"), "a", "b"), - withLabel(newPV("baz"), "a", "c"), + test.PVs( + withLabel(test.NewPV("bar"), "a", "b"), + withLabel(test.NewPV("baz"), "a", "c"), ), }, want: []string{ @@ -222,18 +216,18 @@ func TestBackupResourceFiltering(t *testing.T) { name: "resources with velero.io/exclude-from-backup=true label are not included", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - withLabel(newPod("foo", "bar"), "velero.io/exclude-from-backup", "true"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + withLabel(test.NewPod("foo", "bar"), "velero.io/exclude-from-backup", "true"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - withLabel(newDeployment("zoo", "raz"), "velero.io/exclude-from-backup", "true"), + test.Deployments( + test.NewDeployment("foo", "bar"), + withLabel(test.NewDeployment("zoo", "raz"), "velero.io/exclude-from-backup", "true"), ), - pvs( - withLabel(newPV("bar"), "a", "b"), - withLabel(newPV("baz"), "velero.io/exclude-from-backup", "true"), + test.PVs( + withLabel(test.NewPV("bar"), "a", "b"), + withLabel(test.NewPV("baz"), "velero.io/exclude-from-backup", "true"), ), }, want: []string{ @@ -247,18 +241,18 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). LabelSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"a": "b"}}). Backup(), - apiResources: []*apiResource{ - pods( - withLabel(newPod("foo", "bar"), "velero.io/exclude-from-backup", "true", "a", "b"), - withLabel(newPod("zoo", "raz"), "a", "b"), + apiResources: []*test.APIResource{ + test.Pods( + withLabel(test.NewPod("foo", "bar"), "velero.io/exclude-from-backup", "true", "a", "b"), + withLabel(test.NewPod("zoo", "raz"), "a", "b"), ), - deployments( - newDeployment("foo", "bar"), - withLabel(newDeployment("zoo", "raz"), "velero.io/exclude-from-backup", "true", "a", "b"), + test.Deployments( + test.NewDeployment("foo", "bar"), + withLabel(test.NewDeployment("zoo", "raz"), "velero.io/exclude-from-backup", "true", "a", "b"), ), - pvs( - withLabel(newPV("bar"), "a", "b"), - withLabel(newPV("baz"), "a", "b", "velero.io/exclude-from-backup", "true"), + test.PVs( + withLabel(test.NewPV("bar"), "a", "b"), + withLabel(test.NewPV("baz"), "a", "b", "velero.io/exclude-from-backup", "true"), ), }, want: []string{ @@ -270,18 +264,18 @@ func TestBackupResourceFiltering(t *testing.T) { name: "resources with velero.io/exclude-from-backup label specified but not 'true' are included", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - withLabel(newPod("foo", "bar"), "velero.io/exclude-from-backup", "false"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + withLabel(test.NewPod("foo", "bar"), "velero.io/exclude-from-backup", "false"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - withLabel(newDeployment("zoo", "raz"), "velero.io/exclude-from-backup", "1"), + test.Deployments( + test.NewDeployment("foo", "bar"), + withLabel(test.NewDeployment("zoo", "raz"), "velero.io/exclude-from-backup", "1"), ), - pvs( - withLabel(newPV("bar"), "a", "b"), - withLabel(newPV("baz"), "velero.io/exclude-from-backup", ""), + test.PVs( + withLabel(test.NewPV("bar"), "a", "b"), + withLabel(test.NewPV("baz"), "velero.io/exclude-from-backup", ""), ), }, want: []string{ @@ -299,15 +293,15 @@ func TestBackupResourceFiltering(t *testing.T) { IncludedNamespaces("ns-1", "ns-2"). IncludeClusterResources(true). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-1"), - newPod("ns-3", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-1"), + test.NewPod("ns-3", "pod-1"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, want: []string{ @@ -323,15 +317,15 @@ func TestBackupResourceFiltering(t *testing.T) { IncludedNamespaces("ns-1", "ns-2"). IncludeClusterResources(false). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-1"), - newPod("ns-3", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-1"), + test.NewPod("ns-3", "pod-1"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, want: []string{ @@ -344,15 +338,15 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). IncludedNamespaces("ns-1", "ns-2"). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-1"), - newPod("ns-3", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-1"), + test.NewPod("ns-3", "pod-1"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, want: []string{ @@ -365,15 +359,15 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). IncludeClusterResources(true). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-1"), - newPod("ns-3", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-1"), + test.NewPod("ns-3", "pod-1"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, want: []string{ @@ -389,15 +383,15 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). IncludeClusterResources(false). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-1"), - newPod("ns-3", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-1"), + test.NewPod("ns-3", "pod-1"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, want: []string{ @@ -410,15 +404,15 @@ func TestBackupResourceFiltering(t *testing.T) { name: "should include cluster-scoped resources if backing up all namespaces and IncludeClusterResources=nil", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-1"), - newPod("ns-3", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-1"), + test.NewPod("ns-3", "pod-1"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, want: []string{ @@ -434,14 +428,14 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). IncludedResources("*", "pods"). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + test.Deployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), }, want: []string{ @@ -456,14 +450,14 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). ExcludedResources("*"). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + test.Deployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), }, want: []string{ @@ -478,14 +472,14 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). IncludedResources("pods", "unresolvable"). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + test.Deployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), }, want: []string{ @@ -498,14 +492,14 @@ func TestBackupResourceFiltering(t *testing.T) { backup: defaultBackup(). ExcludedResources("deployments", "unresolvable"). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + test.Deployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), }, want: []string{ @@ -516,9 +510,9 @@ func TestBackupResourceFiltering(t *testing.T) { { name: "terminating resources are not backed up", backup: defaultBackup().Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "ns-2", Name: "pod-2", DeletionTimestamp: &metav1.Time{Time: time.Now()}}}, ), }, @@ -537,7 +531,7 @@ func TestBackupResourceFiltering(t *testing.T) { ) for _, resource := range tc.apiResources { - h.addItems(t, resource.group, resource.version, resource.name, resource.shortName, resource.namespaced, resource.items...) + h.addItems(t, resource) } h.backupper.Backup(h.log, req, backupFile, nil, nil) @@ -555,16 +549,16 @@ func TestBackupResourceCohabitation(t *testing.T) { tests := []struct { name string backup *velerov1.Backup - apiResources []*apiResource + apiResources []*test.APIResource want []string }{ { name: "when deployments exist only in extensions, they're backed up", backup: defaultBackup().Backup(), - apiResources: []*apiResource{ - extensionsDeployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + apiResources: []*test.APIResource{ + test.ExtensionsDeployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), }, want: []string{ @@ -575,14 +569,14 @@ func TestBackupResourceCohabitation(t *testing.T) { { name: "when deployments exist in both apps and extensions, only apps/deployments are backed up", backup: defaultBackup().Backup(), - apiResources: []*apiResource{ - extensionsDeployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + apiResources: []*test.APIResource{ + test.ExtensionsDeployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), - deployments( - newDeployment("foo", "bar"), - newDeployment("zoo", "raz"), + test.Deployments( + test.NewDeployment("foo", "bar"), + test.NewDeployment("zoo", "raz"), ), }, want: []string{ @@ -601,7 +595,7 @@ func TestBackupResourceCohabitation(t *testing.T) { ) for _, resource := range tc.apiResources { - h.addItems(t, resource.group, resource.version, resource.name, resource.shortName, resource.namespaced, resource.items...) + h.addItems(t, resource) } h.backupper.Backup(h.log, req, backupFile, nil, nil) @@ -624,8 +618,8 @@ func TestBackupUsesNewCohabitatingResourcesForEachBackup(t *testing.T) { } backup1File := bytes.NewBuffer([]byte{}) - h.addItems(t, "apps", "v1", "deployments", "deploys", true, newDeployment("ns-1", "deploy-1")) - h.addItems(t, "extensions", "v1", "deployments", "deploys", true, newDeployment("ns-1", "deploy-1")) + h.addItems(t, test.Deployments(test.NewDeployment("ns-1", "deploy-1"))) + h.addItems(t, test.ExtensionsDeployments(test.NewDeployment("ns-1", "deploy-1"))) h.backupper.Backup(h.log, backup1, backup1File, nil, nil) @@ -649,29 +643,29 @@ func TestBackupResourceOrdering(t *testing.T) { tests := []struct { name string backup *velerov1.Backup - apiResources []*apiResource + apiResources []*test.APIResource }{ { name: "core API group: pods come before pvcs, pvcs come before pvs, pvs come before anything else", backup: defaultBackup(). SnapshotVolumes(false). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - pvcs( - newPVC("foo", "bar"), - newPVC("zoo", "raz"), + test.PVCs( + test.NewPVC("foo", "bar"), + test.NewPVC("zoo", "raz"), ), - pvs( - newPV("bar"), - newPV("baz"), + test.PVs( + test.NewPV("bar"), + test.NewPV("baz"), ), - secrets( - newSecret("foo", "bar"), - newSecret("zoo", "raz"), + test.Secrets( + test.NewSecret("foo", "bar"), + test.NewSecret("zoo", "raz"), ), }, }, @@ -686,7 +680,7 @@ func TestBackupResourceOrdering(t *testing.T) { ) for _, resource := range tc.apiResources { - h.addItems(t, resource.group, resource.version, resource.name, resource.shortName, resource.namespaced, resource.items...) + h.addItems(t, resource) } h.backupper.Backup(h.log, req, backupFile, nil, nil) @@ -749,7 +743,7 @@ func TestBackupActionsRunForCorrectItems(t *testing.T) { tests := []struct { name string backup *velerov1.Backup - apiResources []*apiResource + apiResources []*test.APIResource // actions is a map from a recordResourcesAction (which will record the items it was called for) // to a slice of expected items, formatted as {namespace}/{name}. @@ -759,14 +753,14 @@ func TestBackupActionsRunForCorrectItems(t *testing.T) { name: "single action with no selector runs for all items", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-2"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-2"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, actions: map[*recordResourcesAction][]string{ @@ -777,14 +771,14 @@ func TestBackupActionsRunForCorrectItems(t *testing.T) { name: "single action with a resource selector for namespaced resources runs only for matching resources", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-2"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-2"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, actions: map[*recordResourcesAction][]string{ @@ -795,14 +789,14 @@ func TestBackupActionsRunForCorrectItems(t *testing.T) { name: "single action with a resource selector for cluster-scoped resources runs only for matching resources", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-2"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-2"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, actions: map[*recordResourcesAction][]string{ @@ -814,18 +808,18 @@ func TestBackupActionsRunForCorrectItems(t *testing.T) { name: "single action with a namespace selector runs for resources in that namespace plus cluster-scoped resources", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-2"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-2"), ), - pvcs( - newPVC("ns-1", "pvc-1"), - newPVC("ns-2", "pvc-2"), + test.PVCs( + test.NewPVC("ns-1", "pvc-1"), + test.NewPVC("ns-2", "pvc-2"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, actions: map[*recordResourcesAction][]string{ @@ -836,14 +830,14 @@ func TestBackupActionsRunForCorrectItems(t *testing.T) { name: "single action with a resource and namespace selector runs only for matching resources", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-2"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-2"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, actions: map[*recordResourcesAction][]string{ @@ -854,14 +848,14 @@ func TestBackupActionsRunForCorrectItems(t *testing.T) { name: "multiple actions, each with a different resource selector using short name, run for matching resources", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-2"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-2"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, actions: map[*recordResourcesAction][]string{ @@ -873,16 +867,16 @@ func TestBackupActionsRunForCorrectItems(t *testing.T) { name: "actions with selectors that don't match anything don't run for any resources", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), ), - pvcs( - newPVC("ns-2", "pvc-2"), + test.PVCs( + test.NewPVC("ns-2", "pvc-2"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, actions: map[*recordResourcesAction][]string{ @@ -901,7 +895,7 @@ func TestBackupActionsRunForCorrectItems(t *testing.T) { ) for _, resource := range tc.apiResources { - h.addItems(t, resource.group, resource.version, resource.name, resource.shortName, resource.namespaced, resource.items...) + h.addItems(t, resource) } actions := []velero.BackupItemAction{} @@ -929,21 +923,21 @@ func TestBackupWithInvalidActions(t *testing.T) { tests := []struct { name string backup *velerov1.Backup - apiResources []*apiResource + apiResources []*test.APIResource actions []velero.BackupItemAction }{ { name: "action with invalid label selector results in an error", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - pvs( - newPV("bar"), - newPV("baz"), + test.PVs( + test.NewPV("bar"), + test.NewPV("baz"), ), }, actions: []velero.BackupItemAction{ @@ -954,14 +948,14 @@ func TestBackupWithInvalidActions(t *testing.T) { name: "action returning an error from AppliesTo results in an error", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - newPod("foo", "bar"), - newPod("zoo", "raz"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("foo", "bar"), + test.NewPod("zoo", "raz"), ), - pvs( - newPV("bar"), - newPV("baz"), + test.PVs( + test.NewPV("bar"), + test.NewPV("baz"), ), }, actions: []velero.BackupItemAction{ @@ -979,7 +973,7 @@ func TestBackupWithInvalidActions(t *testing.T) { ) for _, resource := range tc.apiResources { - h.addItems(t, resource.group, resource.version, resource.name, resource.shortName, resource.namespaced, resource.items...) + h.addItems(t, resource) } assert.Error(t, h.backupper.Backup(h.log, req, backupFile, tc.actions, nil)) @@ -1025,16 +1019,16 @@ func TestBackupActionModifications(t *testing.T) { tests := []struct { name string backup *velerov1.Backup - apiResources []*apiResource + apiResources []*test.APIResource actions []velero.BackupItemAction want map[string]unstructuredObject }{ { name: "action that adds a label to item gets persisted", backup: defaultBackup().Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), ), }, actions: []velero.BackupItemAction{ @@ -1043,15 +1037,15 @@ func TestBackupActionModifications(t *testing.T) { }), }, want: map[string]unstructuredObject{ - "resources/pods/namespaces/ns-1/pod-1.json": toUnstructuredOrFail(t, withLabel(newPod("ns-1", "pod-1"), "updated", "true")), + "resources/pods/namespaces/ns-1/pod-1.json": toUnstructuredOrFail(t, withLabel(test.NewPod("ns-1", "pod-1"), "updated", "true")), }, }, { name: "action that removes labels from item gets persisted", backup: defaultBackup().Backup(), - apiResources: []*apiResource{ - pods( - withLabel(newPod("ns-1", "pod-1"), "should-be-removed", "true"), + apiResources: []*test.APIResource{ + test.Pods( + withLabel(test.NewPod("ns-1", "pod-1"), "should-be-removed", "true"), ), }, actions: []velero.BackupItemAction{ @@ -1060,15 +1054,15 @@ func TestBackupActionModifications(t *testing.T) { }), }, want: map[string]unstructuredObject{ - "resources/pods/namespaces/ns-1/pod-1.json": toUnstructuredOrFail(t, newPod("ns-1", "pod-1")), + "resources/pods/namespaces/ns-1/pod-1.json": toUnstructuredOrFail(t, test.NewPod("ns-1", "pod-1")), }, }, { name: "action that sets a spec field on item gets persisted", backup: defaultBackup().Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), ), }, actions: []velero.BackupItemAction{ @@ -1077,16 +1071,16 @@ func TestBackupActionModifications(t *testing.T) { }), }, want: map[string]unstructuredObject{ - "resources/pods/namespaces/ns-1/pod-1.json": toUnstructuredOrFail(t, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "ns-1", Name: "pod-1"}, Spec: corev1.PodSpec{NodeName: "foo"}}), + "resources/pods/namespaces/ns-1/pod-1.json": toUnstructuredOrFail(t, &corev1.Pod{TypeMeta: metav1.TypeMeta{Kind: "Pod", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Namespace: "ns-1", Name: "pod-1"}, Spec: corev1.PodSpec{NodeName: "foo"}}), }, }, { name: "modifications to name and namespace in an action are persisted in JSON and in filename", backup: defaultBackup(). Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), ), }, actions: []velero.BackupItemAction{ @@ -1096,7 +1090,7 @@ func TestBackupActionModifications(t *testing.T) { }), }, want: map[string]unstructuredObject{ - "resources/pods/namespaces/ns-1-updated/pod-1-updated.json": toUnstructuredOrFail(t, newPod("ns-1-updated", "pod-1-updated")), + "resources/pods/namespaces/ns-1-updated/pod-1-updated.json": toUnstructuredOrFail(t, test.NewPod("ns-1-updated", "pod-1-updated")), }, }, } @@ -1110,7 +1104,7 @@ func TestBackupActionModifications(t *testing.T) { ) for _, resource := range tc.apiResources { - h.addItems(t, resource.group, resource.version, resource.name, resource.shortName, resource.namespaced, resource.items...) + h.addItems(t, resource) } err := h.backupper.Backup(h.log, req, backupFile, tc.actions, nil) @@ -1130,18 +1124,18 @@ func TestBackupActionAdditionalItems(t *testing.T) { tests := []struct { name string backup *velerov1.Backup - apiResources []*apiResource + apiResources []*test.APIResource actions []velero.BackupItemAction want []string }{ { name: "additional items that are already being backed up are not backed up twice", backup: defaultBackup().Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-2"), - newPod("ns-3", "pod-3"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-2"), + test.NewPod("ns-3", "pod-3"), ), }, actions: []velero.BackupItemAction{ @@ -1166,11 +1160,11 @@ func TestBackupActionAdditionalItems(t *testing.T) { { name: "when using a backup namespace filter, additional items that are in a non-included namespace are not backed up", backup: defaultBackup().IncludedNamespaces("ns-1").Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-2"), - newPod("ns-3", "pod-3"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-2"), + test.NewPod("ns-3", "pod-3"), ), }, actions: []velero.BackupItemAction{ @@ -1192,14 +1186,14 @@ func TestBackupActionAdditionalItems(t *testing.T) { { name: "when using a backup namespace filter, additional items that are cluster-scoped are backed up", backup: defaultBackup().IncludedNamespaces("ns-1").Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-2"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-2"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, actions: []velero.BackupItemAction{ @@ -1223,13 +1217,13 @@ func TestBackupActionAdditionalItems(t *testing.T) { { name: "when using a backup resource filter, additional items that are non-included resources are not backed up", backup: defaultBackup().IncludedResources("pods").Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, actions: []velero.BackupItemAction{ @@ -1251,14 +1245,14 @@ func TestBackupActionAdditionalItems(t *testing.T) { { name: "when IncludeClusterResources=false, additional items that are cluster-scoped are not backed up", backup: defaultBackup().IncludeClusterResources(false).Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-2"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-2"), ), - pvs( - newPV("pv-1"), - newPV("pv-2"), + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, actions: []velero.BackupItemAction{ @@ -1281,11 +1275,11 @@ func TestBackupActionAdditionalItems(t *testing.T) { { name: "if there's an error backing up additional items, the item the action was run for isn't backed up", backup: defaultBackup().Backup(), - apiResources: []*apiResource{ - pods( - newPod("ns-1", "pod-1"), - newPod("ns-2", "pod-2"), - newPod("ns-3", "pod-3"), + apiResources: []*test.APIResource{ + test.Pods( + test.NewPod("ns-1", "pod-1"), + test.NewPod("ns-2", "pod-2"), + test.NewPod("ns-3", "pod-3"), ), }, actions: []velero.BackupItemAction{ @@ -1317,7 +1311,7 @@ func TestBackupActionAdditionalItems(t *testing.T) { ) for _, resource := range tc.apiResources { - h.addItems(t, resource.group, resource.version, resource.name, resource.shortName, resource.namespaced, resource.items...) + h.addItems(t, resource) } err := h.backupper.Backup(h.log, req, backupFile, tc.actions, nil) @@ -1463,7 +1457,7 @@ func TestBackupWithSnapshots(t *testing.T) { name string req *Request vsls []*velerov1.VolumeSnapshotLocation - apiResources []*apiResource + apiResources []*test.APIResource snapshotterGetter volumeSnapshotterGetter want []*volume.Snapshot }{ @@ -1475,9 +1469,9 @@ func TestBackupWithSnapshots(t *testing.T) { newSnapshotLocation("velero", "default", "default"), }, }, - apiResources: []*apiResource{ - pvs( - newPV("pv-1"), + apiResources: []*test.APIResource{ + test.PVs( + test.NewPV("pv-1"), ), }, snapshotterGetter: map[string]velero.VolumeSnapshotter{ @@ -1508,9 +1502,9 @@ func TestBackupWithSnapshots(t *testing.T) { newSnapshotLocation("velero", "default", "default"), }, }, - apiResources: []*apiResource{ - pvs( - withLabel(newPV("pv-1"), "failure-domain.beta.kubernetes.io/zone", "zone-1"), + apiResources: []*test.APIResource{ + test.PVs( + withLabel(test.NewPV("pv-1"), "failure-domain.beta.kubernetes.io/zone", "zone-1"), ), }, snapshotterGetter: map[string]velero.VolumeSnapshotter{ @@ -1542,9 +1536,9 @@ func TestBackupWithSnapshots(t *testing.T) { newSnapshotLocation("velero", "default", "default"), }, }, - apiResources: []*apiResource{ - pvs( - newPV("pv-1"), + apiResources: []*test.APIResource{ + test.PVs( + test.NewPV("pv-1"), ), }, snapshotterGetter: map[string]velero.VolumeSnapshotter{ @@ -1574,9 +1568,9 @@ func TestBackupWithSnapshots(t *testing.T) { newSnapshotLocation("velero", "default", "default"), }, }, - apiResources: []*apiResource{ - pvs( - newPV("pv-1"), + apiResources: []*test.APIResource{ + test.PVs( + test.NewPV("pv-1"), ), }, snapshotterGetter: map[string]velero.VolumeSnapshotter{ @@ -1589,9 +1583,9 @@ func TestBackupWithSnapshots(t *testing.T) { req: &Request{ Backup: defaultBackup().Backup(), }, - apiResources: []*apiResource{ - pvs( - newPV("pv-1"), + apiResources: []*test.APIResource{ + test.PVs( + test.NewPV("pv-1"), ), }, snapshotterGetter: map[string]velero.VolumeSnapshotter{ @@ -1607,9 +1601,9 @@ func TestBackupWithSnapshots(t *testing.T) { newSnapshotLocation("velero", "default", "default"), }, }, - apiResources: []*apiResource{ - pvs( - newPV("pv-1"), + apiResources: []*test.APIResource{ + test.PVs( + test.NewPV("pv-1"), ), }, snapshotterGetter: map[string]velero.VolumeSnapshotter{}, @@ -1623,9 +1617,9 @@ func TestBackupWithSnapshots(t *testing.T) { newSnapshotLocation("velero", "default", "default"), }, }, - apiResources: []*apiResource{ - pvs( - newPV("pv-1"), + apiResources: []*test.APIResource{ + test.PVs( + test.NewPV("pv-1"), ), }, snapshotterGetter: map[string]velero.VolumeSnapshotter{ @@ -1642,10 +1636,10 @@ func TestBackupWithSnapshots(t *testing.T) { newSnapshotLocation("velero", "another", "another"), }, }, - apiResources: []*apiResource{ - pvs( - newPV("pv-1"), - newPV("pv-2"), + apiResources: []*test.APIResource{ + test.PVs( + test.NewPV("pv-1"), + test.NewPV("pv-2"), ), }, snapshotterGetter: map[string]velero.VolumeSnapshotter{ @@ -1693,7 +1687,7 @@ func TestBackupWithSnapshots(t *testing.T) { ) for _, resource := range tc.apiResources { - h.addItems(t, resource.group, resource.version, resource.name, resource.shortName, resource.namespaced, resource.items...) + h.addItems(t, resource) } err := h.backupper.Backup(h.log, tc.req, backupFile, nil, tc.snapshotterGetter) @@ -1723,112 +1717,28 @@ func (a *pluggableAction) AppliesTo() (velero.ResourceSelector, error) { return a.selector, nil } -type apiResource struct { - group string - version string - name string - shortName string - namespaced bool - items []metav1.Object -} - -func pods(items ...metav1.Object) *apiResource { - return &apiResource{ - group: "", - version: "v1", - name: "pods", - shortName: "po", - namespaced: true, - items: items, - } -} - -func pvcs(items ...metav1.Object) *apiResource { - return &apiResource{ - group: "", - version: "v1", - name: "persistentvolumeclaims", - shortName: "pvc", - namespaced: true, - items: items, - } -} - -func secrets(items ...metav1.Object) *apiResource { - return &apiResource{ - group: "", - version: "v1", - name: "secrets", - shortName: "secrets", - namespaced: true, - items: items, - } -} - -func deployments(items ...metav1.Object) *apiResource { - return &apiResource{ - group: "apps", - version: "v1", - name: "deployments", - shortName: "deploy", - namespaced: true, - items: items, - } -} - -func extensionsDeployments(items ...metav1.Object) *apiResource { - return &apiResource{ - group: "extensions", - version: "v1", - name: "deployments", - shortName: "deploy", - namespaced: true, - items: items, - } -} - -func pvs(items ...metav1.Object) *apiResource { - return &apiResource{ - group: "", - version: "v1", - name: "persistentvolumes", - shortName: "pv", - namespaced: false, - items: items, - } -} - type harness struct { - veleroClient *fake.Clientset - kubeClient *kubefake.Clientset - dynamicClient *dynamicfake.FakeDynamicClient - discoveryClient *test.DiscoveryClient - backupper *kubernetesBackupper - log logrus.FieldLogger + *test.APIServer + backupper *kubernetesBackupper + log logrus.FieldLogger } -func (h *harness) addItems(t *testing.T, group, version, resource, shortName string, namespaced bool, items ...metav1.Object) { +func (h *harness) addItems(t *testing.T, resource *test.APIResource) { t.Helper() - h.discoveryClient.WithResource(group, version, resource, namespaced, shortName) + h.DiscoveryClient.WithAPIResource(resource) require.NoError(t, h.backupper.discoveryHelper.Refresh()) - gvr := schema.GroupVersionResource{ - Group: group, - Version: version, - Resource: resource, - } - - for _, item := range items { + for _, item := range resource.Items { obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(item) require.NoError(t, err) unstructuredObj := &unstructured.Unstructured{Object: obj} - if namespaced { - _, err = h.dynamicClient.Resource(gvr).Namespace(item.GetNamespace()).Create(unstructuredObj, metav1.CreateOptions{}) + if resource.Namespaced { + _, err = h.DynamicClient.Resource(resource.GVR()).Namespace(item.GetNamespace()).Create(unstructuredObj, metav1.CreateOptions{}) } else { - _, err = h.dynamicClient.Resource(gvr).Create(unstructuredObj, metav1.CreateOptions{}) + _, err = h.DynamicClient.Resource(resource.GVR()).Create(unstructuredObj, metav1.CreateOptions{}) } require.NoError(t, err) } @@ -1837,26 +1747,16 @@ func (h *harness) addItems(t *testing.T, group, version, resource, shortName str func newHarness(t *testing.T) *harness { t.Helper() - // API server fakes - var ( - veleroClient = fake.NewSimpleClientset() - kubeClient = kubefake.NewSimpleClientset() - dynamicClient = dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) - discoveryClient = &test.DiscoveryClient{FakeDiscovery: kubeClient.Discovery().(*discoveryfake.FakeDiscovery)} - ) - + apiServer := test.NewAPIServer(t) log := logrus.StandardLogger() - discoveryHelper, err := discovery.NewHelper(discoveryClient, log) + discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, log) require.NoError(t, err) return &harness{ - veleroClient: veleroClient, - kubeClient: kubeClient, - dynamicClient: dynamicClient, - discoveryClient: discoveryClient, + APIServer: apiServer, backupper: &kubernetesBackupper{ - dynamicFactory: client.NewDynamicFactory(dynamicClient), + dynamicFactory: client.NewDynamicFactory(apiServer.DynamicClient), discoveryHelper: discoveryHelper, groupBackupperFactory: new(defaultGroupBackupperFactory), @@ -1885,50 +1785,6 @@ func withLabel(obj metav1.Object, labelPairs ...string) metav1.Object { return obj } -func newPod(ns, name string) *corev1.Pod { - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: name, - }, - } -} - -func newPVC(ns, name string) *corev1.PersistentVolumeClaim { - return &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: name, - }, - } -} - -func newSecret(ns, name string) *corev1.Secret { - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: name, - }, - } -} - -func newDeployment(ns, name string) *appsv1.Deployment { - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: name, - }, - } -} - -func newPV(name string) *corev1.PersistentVolume { - return &corev1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - } -} - func newSnapshotLocation(ns, name, provider string) *velerov1.VolumeSnapshotLocation { return &velerov1.VolumeSnapshotLocation{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/test/api_server.go b/pkg/test/api_server.go new file mode 100644 index 00000000000..90c3a483b05 --- /dev/null +++ b/pkg/test/api_server.go @@ -0,0 +1,57 @@ +/* +Copyright 2019 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "testing" + + "k8s.io/apimachinery/pkg/runtime" + discoveryfake "k8s.io/client-go/discovery/fake" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + + "github.com/heptio/velero/pkg/generated/clientset/versioned/fake" +) + +// APIServer contains in-memory fakes for all of the relevant +// Kubernetes API server clients. +type APIServer struct { + VeleroClient *fake.Clientset + KubeClient *kubefake.Clientset + DynamicClient *dynamicfake.FakeDynamicClient + DiscoveryClient *DiscoveryClient +} + +// NewAPIServer constructs an APIServer with all of its clients +// initialized. +func NewAPIServer(t *testing.T) *APIServer { + t.Helper() + + var ( + veleroClient = fake.NewSimpleClientset() + kubeClient = kubefake.NewSimpleClientset() + dynamicClient = dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + discoveryClient = &DiscoveryClient{FakeDiscovery: kubeClient.Discovery().(*discoveryfake.FakeDiscovery)} + ) + + return &APIServer{ + VeleroClient: veleroClient, + KubeClient: kubeClient, + DynamicClient: dynamicClient, + DiscoveryClient: discoveryClient, + } +} diff --git a/pkg/test/discovery_client.go b/pkg/test/discovery_client.go index 5a5afbfeca5..fe06af635ea 100644 --- a/pkg/test/discovery_client.go +++ b/pkg/test/discovery_client.go @@ -38,10 +38,11 @@ func (c *DiscoveryClient) ServerPreferredResources() ([]*metav1.APIResourceList, // TEST HELPERS // -func (c *DiscoveryClient) WithResource(group, version, resource string, namespaced bool, shortNames ...string) *DiscoveryClient { +// WithAPIResource adds the API resource to the discovery client. +func (c *DiscoveryClient) WithAPIResource(resource *APIResource) *DiscoveryClient { gv := metav1.GroupVersion{ - Group: group, - Version: version, + Group: resource.Group, + Version: resource.Version, } var resourceList *metav1.APIResourceList @@ -61,20 +62,20 @@ func (c *DiscoveryClient) WithResource(group, version, resource string, namespac } for _, itm := range resourceList.APIResources { - if itm.Name == resource { + if itm.Name == resource.Name { return c } } resourceList.APIResources = append(resourceList.APIResources, metav1.APIResource{ - Name: resource, - SingularName: strings.TrimSuffix(resource, "s"), - Namespaced: namespaced, - Group: group, - Version: version, - Kind: strings.Title(strings.TrimSuffix(resource, "s")), + Name: resource.Name, + SingularName: strings.TrimSuffix(resource.Name, "s"), + Namespaced: resource.Namespaced, + Group: resource.Group, + Version: resource.Version, + Kind: strings.Title(strings.TrimSuffix(resource.Name, "s")), Verbs: metav1.Verbs([]string{"list", "create", "get", "delete"}), - ShortNames: shortNames, + ShortNames: []string{resource.ShortName}, }) return c diff --git a/pkg/test/resources.go b/pkg/test/resources.go new file mode 100644 index 00000000000..f40fc2c5cdf --- /dev/null +++ b/pkg/test/resources.go @@ -0,0 +1,168 @@ +/* +Copyright 2019 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// APIResource stores information about a specific Kubernetes API +// resource. +type APIResource struct { + Group string + Version string + Name string + ShortName string + Namespaced bool + Items []metav1.Object +} + +// GVR returns a GroupVersionResource representing the resource. +func (r *APIResource) GVR() schema.GroupVersionResource { + return schema.GroupVersionResource{ + Group: r.Group, + Version: r.Version, + Resource: r.Name, + } +} + +// Pods returns an APIResource describing core/v1's Pods. +func Pods(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "", + Version: "v1", + Name: "pods", + ShortName: "po", + Namespaced: true, + Items: items, + } +} + +func PVCs(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "", + Version: "v1", + Name: "persistentvolumeclaims", + ShortName: "pvc", + Namespaced: true, + Items: items, + } +} + +func PVs(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "", + Version: "v1", + Name: "persistentvolumes", + ShortName: "pv", + Namespaced: false, + Items: items, + } +} + +func Secrets(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "", + Version: "v1", + Name: "secrets", + ShortName: "secrets", + Namespaced: true, + Items: items, + } +} + +func Deployments(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "apps", + Version: "v1", + Name: "deployments", + ShortName: "deploy", + Namespaced: true, + Items: items, + } +} + +func ExtensionsDeployments(items ...metav1.Object) *APIResource { + return &APIResource{ + Group: "extensions", + Version: "v1", + Name: "deployments", + ShortName: "deploy", + Namespaced: true, + Items: items, + } +} + +func NewPod(ns, name string) *corev1.Pod { + return &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: objectMeta(ns, name), + } +} + +func NewPVC(ns, name string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + }, + ObjectMeta: objectMeta(ns, name), + } +} + +func NewPV(name string) *corev1.PersistentVolume { + return &corev1.PersistentVolume{ + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolume", + APIVersion: "v1", + }, + ObjectMeta: objectMeta("", name), + } +} + +func NewSecret(ns, name string) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: objectMeta(ns, name), + } +} + +func NewDeployment(ns, name string) *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: objectMeta(ns, name), + } +} + +func objectMeta(ns, name string) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Namespace: ns, + Name: name, + } +} From d421fcd85c73146db88baccdfd9ecc3e45ffb083 Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Thu, 20 Jun 2019 09:06:58 -0600 Subject: [PATCH 008/118] initial structure for refactored pkg/restore tests Signed-off-by: Steve Kriss --- pkg/restore/builder.go | 78 +++++++++++ pkg/restore/restore_new_test.go | 232 ++++++++++++++++++++++++++++++++ pkg/restore/restore_test.go | 12 -- 3 files changed, 310 insertions(+), 12 deletions(-) create mode 100644 pkg/restore/builder.go create mode 100644 pkg/restore/restore_new_test.go diff --git a/pkg/restore/builder.go b/pkg/restore/builder.go new file mode 100644 index 00000000000..2daf1afd641 --- /dev/null +++ b/pkg/restore/builder.go @@ -0,0 +1,78 @@ +/* +Copyright 2019 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + velerov1api "github.com/heptio/velero/pkg/apis/velero/v1" +) + +// Builder is a helper for concisely constructing Restore API objects. +type Builder struct { + restore velerov1api.Restore +} + +// NewBuilder returns a Builder for a Restore with no namespace/name. +func NewBuilder() *Builder { + return NewNamedBuilder("", "") +} + +// NewNamedBuilder returns a Builder for a Restore with the specified namespace +// and name. +func NewNamedBuilder(namespace, name string) *Builder { + return &Builder{ + restore: velerov1api.Restore{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov1api.SchemeGroupVersion.String(), + Kind: "Restore", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + }, + } +} + +// Restore returns the built Restore API object. +func (b *Builder) Restore() *velerov1api.Restore { + return &b.restore +} + +// Backup sets the Restore's backup name. +func (b *Builder) Backup(name string) *Builder { + b.restore.Spec.BackupName = name + return b +} + +// NamespaceMappings sets the Restore's namespace mappings. +func (b *Builder) NamespaceMappings(mapping ...string) *Builder { + if b.restore.Spec.NamespaceMapping == nil { + b.restore.Spec.NamespaceMapping = make(map[string]string) + } + + if len(mapping)%2 != 0 { + panic("mapping must contain an even number of values") + } + + for i := 0; i < len(mapping); i += 2 { + b.restore.Spec.NamespaceMapping[mapping[i]] = mapping[i+1] + } + + return b +} diff --git a/pkg/restore/restore_new_test.go b/pkg/restore/restore_new_test.go new file mode 100644 index 00000000000..8197671a756 --- /dev/null +++ b/pkg/restore/restore_new_test.go @@ -0,0 +1,232 @@ +/* +Copyright 2019 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "fmt" + "io" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + + velerov1api "github.com/heptio/velero/pkg/apis/velero/v1" + "github.com/heptio/velero/pkg/backup" + "github.com/heptio/velero/pkg/client" + "github.com/heptio/velero/pkg/discovery" + "github.com/heptio/velero/pkg/test" + "github.com/heptio/velero/pkg/util/encode" + testutil "github.com/heptio/velero/pkg/util/test" +) + +func TestRestoreNew(t *testing.T) { + tests := []struct { + name string + restore *velerov1api.Restore + backup *velerov1api.Backup + apiResources []*test.APIResource + tarball io.Reader + want map[*test.APIResource][]string + }{ + { + name: "base case - restore a single resource", + restore: defaultRestore().Backup("backup-1").Restore(), + backup: backup.NewNamedBuilder(velerov1api.DefaultNamespace, "backup-1").Backup(), + tarball: newTarWriter(t). + add("metadata/version", []byte("1")). + add("resources/pods/namespaces/ns-1/pod-1.json", test.NewPod("ns-1", "pod-1")). + done(), + apiResources: []*test.APIResource{ + test.Pods(), + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-1/pod-1"}, + }, + }, + { + name: "restore a resource to a remapped namespace", + restore: defaultRestore().Backup("backup-1").NamespaceMappings("ns-1", "ns-2").Restore(), + backup: backup.NewNamedBuilder(velerov1api.DefaultNamespace, "backup-1").Backup(), + tarball: newTarWriter(t). + add("metadata/version", []byte("1")). + add("resources/pods/namespaces/ns-1/pod-1.json", test.NewPod("ns-1", "pod-1")). + done(), + apiResources: []*test.APIResource{ + test.Pods(), + }, + want: map[*test.APIResource][]string{ + test.Pods(): {"ns-2/pod-1"}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + h := newHarness(t) + + for _, r := range tc.apiResources { + h.DiscoveryClient.WithAPIResource(r) + } + require.NoError(t, h.restorer.discoveryHelper.Refresh()) + + warnings, errs := h.restorer.Restore( + h.log, + tc.restore, + tc.backup, + nil, // volume snapshots + tc.tarball, + nil, // actions + nil, // snapshot location lister + nil, // volume snapshotter getter + ) + + assertEmptyResults(t, warnings, errs) + assertAPIContents(t, h, tc.want) + }) + } +} + +func defaultRestore() *Builder { + return NewNamedBuilder(velerov1api.DefaultNamespace, "restore-1") +} + +// assertAPIContents asserts that the dynamic client on the provided harness contains +// all of the items specified in 'want' (a map from an APIResource definition to a slice +// of resource identifiers, formatted as /). +func assertAPIContents(t *testing.T, h *harness, want map[*test.APIResource][]string) { + for r, want := range want { + res, err := h.DynamicClient.Resource(r.GVR()).List(metav1.ListOptions{}) + assert.NoError(t, err) + if err != nil { + continue + } + + got := sets.NewString() + for _, item := range res.Items { + got.Insert(fmt.Sprintf("%s/%s", item.GetNamespace(), item.GetName())) + } + + assert.Equal(t, sets.NewString(want...), got) + } +} + +func assertEmptyResults(t *testing.T, res ...Result) { + t.Helper() + + for _, r := range res { + assert.Empty(t, r.Cluster) + assert.Empty(t, r.Namespaces) + assert.Empty(t, r.Velero) + } +} + +type tarWriter struct { + t *testing.T + buf *bytes.Buffer + gzw *gzip.Writer + tw *tar.Writer +} + +func newTarWriter(t *testing.T) *tarWriter { + tw := new(tarWriter) + tw.t = t + tw.buf = new(bytes.Buffer) + tw.gzw = gzip.NewWriter(tw.buf) + tw.tw = tar.NewWriter(tw.gzw) + + return tw +} + +func (tw *tarWriter) add(name string, obj interface{}) *tarWriter { + tw.t.Helper() + + var data []byte + var err error + + switch obj.(type) { + case runtime.Object: + data, err = encode.Encode(obj.(runtime.Object), "json") + case []byte: + data = obj.([]byte) + default: + data, err = json.Marshal(obj) + } + require.NoError(tw.t, err) + + require.NoError(tw.t, tw.tw.WriteHeader(&tar.Header{ + Name: name, + Size: int64(len(data)), + Typeflag: tar.TypeReg, + Mode: 0755, + ModTime: time.Now(), + })) + + _, err = tw.tw.Write(data) + require.NoError(tw.t, err) + + return tw +} + +func (tw *tarWriter) done() *bytes.Buffer { + require.NoError(tw.t, tw.tw.Close()) + require.NoError(tw.t, tw.gzw.Close()) + + return tw.buf +} + +type harness struct { + *test.APIServer + + restorer *kubernetesRestorer + log logrus.FieldLogger +} + +func newHarness(t *testing.T) *harness { + t.Helper() + + apiServer := test.NewAPIServer(t) + log := logrus.StandardLogger() + + discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, log) + require.NoError(t, err) + + return &harness{ + APIServer: apiServer, + restorer: &kubernetesRestorer{ + discoveryHelper: discoveryHelper, + dynamicFactory: client.NewDynamicFactory(apiServer.DynamicClient), + namespaceClient: apiServer.KubeClient.CoreV1().Namespaces(), + resourceTerminatingTimeout: time.Minute, + logger: log, + fileSystem: testutil.NewFakeFileSystem(), + + // unsupported + resticRestorerFactory: nil, + resticTimeout: 0, + }, + log: log, + } +} diff --git a/pkg/restore/restore_test.go b/pkg/restore/restore_test.go index c7ff983465b..4e7034b648e 100644 --- a/pkg/restore/restore_test.go +++ b/pkg/restore/restore_test.go @@ -36,7 +36,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes/scheme" - corev1 "k8s.io/client-go/kubernetes/typed/core/v1" api "github.com/heptio/velero/pkg/apis/velero/v1" pkgclient "github.com/heptio/velero/pkg/client" @@ -1962,14 +1961,3 @@ func (r *fakeAction) Execute(input *velero.RestoreItemActionExecuteInput) (*vele return velero.NewRestoreItemActionExecuteOutput(res), nil } - -type fakeNamespaceClient struct { - createdNamespaces []*v1.Namespace - - corev1.NamespaceInterface -} - -func (nsc *fakeNamespaceClient) Create(ns *v1.Namespace) (*v1.Namespace, error) { - nsc.createdNamespaces = append(nsc.createdNamespaces, ns) - return ns, nil -} From d916ae0a250d0f6d263185ece796d83296d5ef84 Mon Sep 17 00:00:00 2001 From: Mickey Boxell Date: Fri, 21 Jun 2019 09:29:58 -0700 Subject: [PATCH 009/118] document Oracle Cloud Storage as a backup provider (#1570) Signed-off-by: mickey.boxell --- site/docs/master/oracle-config.md | 245 +++++++++++++++++++++++++++++ site/docs/master/support-matrix.md | 2 + site/docs/v1.0.0/oracle-config.md | 245 +++++++++++++++++++++++++++++ site/docs/v1.0.0/support-matrix.md | 2 + 4 files changed, 494 insertions(+) create mode 100644 site/docs/master/oracle-config.md create mode 100644 site/docs/v1.0.0/oracle-config.md diff --git a/site/docs/master/oracle-config.md b/site/docs/master/oracle-config.md new file mode 100644 index 00000000000..089fa65c40d --- /dev/null +++ b/site/docs/master/oracle-config.md @@ -0,0 +1,245 @@ +# Use Oracle Cloud as a Backup Storage Provider for Velero + +## Introduction + +[Velero](https://velero.io/) is a tool used to backup and migrate Kubernetes applications. Here are the steps to use [Oracle Cloud Object Storage](https://docs.cloud.oracle.com/iaas/Content/Object/Concepts/objectstorageoverview.htm) as a destination for Velero backups. + +1. [Download Velero](#download-velero) +2. [Create A Customer Secret Key](#create-a-customer-secret-key) +3. [Create An Oracle Object Storage Bucket](#create-an-oracle-object-storage-bucket) +4. [Install Velero](#install-velero) +5. [Clean Up](#clean-up) +6. [Examples](#examples) +7. [Additional Reading](#additional-reading) + +## Download Velero + +1. Download the [latest release](https://github.com/heptio/velero/releases/) of Velero to your development environment. This includes the `velero` CLI utility and example Kubernetes manifest files. For example: + + ``` + wget https://github.com/heptio/velero/releases/download/v1.0.0/velero-v1.0.0-linux-amd64.tar.gz + ``` + + *We strongly recommend that you use an official release of Velero. The tarballs for each release contain the velero command-line client. The code in the master branch of the Velero repository is under active development and is not guaranteed to be stable!* + +2. Untar the release in your `/usr/bin` directory: `tar -xzvf .tar.gz` + + You may choose to rename the directory `velero` for the sake of simplicty: `mv velero-v1.0.0-linux-amd64 velero` + +3. Add it to your PATH: `export PATH=/usr/local/bin/velero:$PATH` + +4. Run `velero` to confirm the CLI has been installed correctly. You should see an output like this: + +``` +$ velero +Velero is a tool for managing disaster recovery, specifically for Kubernetes +cluster resources. It provides a simple, configurable, and operationally robust +way to back up your application state and associated data. + +If you're familiar with kubectl, Velero supports a similar model, allowing you to +execute commands such as 'velero get backup' and 'velero create schedule'. The same +operations can also be performed as 'velero backup get' and 'velero schedule create'. + +Usage: + velero [command] +``` + + + +## Create A Customer Secret Key + +1. Oracle Object Storage provides an API to enable interoperability with Amazon S3. To use this Amazon S3 Compatibility API, you need to generate the signing key required to authenticate with Amazon S3. This special signing key is an Access Key/Secret Key pair. Follow these steps to [create a Customer Secret Key](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcredentials.htm#To4). Refer to this link for more information about [Working with Customer Secret Keys](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcredentials.htm#s3). + +2. Create a Velero credentials file with your Customer Secret Key: + + ``` + $ vi credentials-velero + + [default] + aws_access_key_id=bae031188893d1eb83719648790ac850b76c9441 + aws_secret_access_key=MmY9heKrWiNVCSZQ2Mf5XTJ6Ys93Bw2d2D6NMSTXZlk= + ``` + + + +## Create An Oracle Object Storage Bucket + +Create an Oracle Cloud Object Storage bucket called `velero` in the root compartment of your Oracle Cloud tenancy. Refer to this page for [more information about creating a bucket with Object Storage](https://docs.cloud.oracle.com/iaas/Content/Object/Tasks/managingbuckets.htm#usingconsole). + + + +## Install Velero + +You will need the following information to install Velero into your Kubernetes cluster with Oracle Object Storage as the Backup Storage provider: + +``` +velero install \ + --provider [provider name] \ + --bucket [bucket name] \ + --prefix [tenancy name] \ + --use-volume-snapshots=false \ + --secret-file [secret file location] \ + --backup-location-config region=[region],s3ForcePathStyle="true",s3Url=[storage API endpoint] +``` + +- `--provider` Because we are using the S3-compatible API, we will use `aws` as our provider. +- `--bucket` The name of the bucket created in Oracle Object Storage - in our case this is named `velero`. +- ` --prefix` The name of your Oracle Cloud tenancy - in our case this is named `oracle-cloudnative`. +- `--use-volume-snapshots=false` Velero does not currently have a volume snapshot plugin for Oracle Cloud creating volume snapshots is disabled. +- `--secret-file` The path to your `credentials-velero` file. +- `--backup-location-config` The path to your Oracle Object Storage bucket. This consists of your `region` which corresponds to your Oracle Cloud region name ([List of Oracle Cloud Regions](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/regions.htm?Highlight=regions)) and the `s3Url`, the S3-compatible API endpoint for Oracle Object Storage based on your region: `https://oracle-cloudnative.compat.objectstorage.[region name].oraclecloud.com` + +For example: + +``` +velero install \ + --provider aws \ + --bucket velero \ + --prefix oracle-cloudnative \ + --use-volume-snapshots=false \ + --secret-file /Users/mboxell/bin/velero/credentials-velero \ + --backup-location-config region=us-phoenix-1,s3ForcePathStyle="true",s3Url=https://oracle-cloudnative.compat.objectstorage.us-phoenix-1.oraclecloud.com +``` + +This will create a `velero` namespace in your cluster along with a number of CRDs, a ClusterRoleBinding, ServiceAccount, Secret, and Deployment for Velero. If your pod fails to successfully provision, you can troubleshoot your installation by running: `kubectl logs [velero pod name]`. + + + +## Clean Up + +To remove Velero from your environment, delete the namespace, ClusterRoleBinding, ServiceAccount, Secret, and Deployment and delete the CRDs, run: + +``` +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +``` + +This will remove all resources created by `velero install`. + + + +## Examples + +After creating the Velero server in your cluster, try this example: + +### Basic example (without PersistentVolumes) + +1. Start the sample nginx app: `kubectl apply -f examples/nginx-app/base.yaml` + + This will create an `nginx-example` namespace with a `nginx-deployment` deployment, and `my-nginx` service. + + ``` + $ kubectl apply -f examples/nginx-app/base.yaml + namespace/nginx-example created + deployment.apps/nginx-deployment created + service/my-nginx created + ``` + + You can see the created resources by running `kubectl get all` + + ``` + $ kubectl get all + NAME READY STATUS RESTARTS AGE + pod/nginx-deployment-67594d6bf6-4296p 1/1 Running 0 20s + pod/nginx-deployment-67594d6bf6-f9r5s 1/1 Running 0 20s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/my-nginx LoadBalancer 10.96.69.166 80:31859/TCP 21s + + NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE + deployment.apps/nginx-deployment 2 2 2 2 21s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/nginx-deployment-67594d6bf6 2 2 2 21s + ``` + +2. Create a backup: `velero backup create nginx-backup --include-namespaces nginx-example` + + ``` + $ velero backup create nginx-backup --include-namespaces nginx-example + Backup request "nginx-backup" submitted successfully. + Run `velero backup describe nginx-backup` or `velero backup logs nginx-backup` for more details. + ``` + + At this point you can navigate to appropriate bucket, which we called `velero`, in the Oracle Cloud Object Storage console to see the resources backed up using Velero. + +3. Simulate a disaster by deleting the `nginx-example` namespace: `kubectl delete namespaces nginx-example` + + ``` + $ kubectl delete namespaces nginx-example + namespace "nginx-example" deleted + ``` + + Wait for the namespace to be deleted. To check that the nginx deployment, service, and namespace are gone, run: + + ``` + kubectl get deployments --namespace=nginx-example + kubectl get services --namespace=nginx-example + kubectl get namespace/nginx-example + ``` + + This should return: `No resources found.` + +4. Restore your lost resources: `velero restore create --from-backup nginx-backup` + + ``` + $ velero restore create --from-backup nginx-backup + Restore request "nginx-backup-20190604102710" submitted successfully. + Run `velero restore describe nginx-backup-20190604102710` or `velero restore logs nginx-backup-20190604102710` for more details. + ``` + + Running `kubectl get namespaces` will show that the `nginx-example` namespace has been restored along with its contents. + +5. Run: `velero restore get` to view the list of restored resources. After the restore finishes, the output looks like the following: + + ``` + $ velero restore get + NAME BACKUP STATUS WARNINGS ERRORS CREATED SELECTOR + nginx-backup-20190604104249 nginx-backup Completed 0 0 2019-06-04 10:42:39 -0700 PDT + ``` + + NOTE: The restore can take a few moments to finish. During this time, the `STATUS` column reads `InProgress`. + + After a successful restore, the `STATUS` column shows `Completed`, and `WARNINGS` and `ERRORS` will show `0`. All objects in the `nginx-example` namespace should be just as they were before you deleted them. + + If there are errors or warnings, for instance if the `STATUS` column displays `FAILED` instead of `InProgress`, you can look at them in detail with `velero restore describe ` + + +6. Clean up the environment with `kubectl delete -f examples/nginx-app/base.yaml` + + ``` + $ kubectl delete -f examples/nginx-app/base.yaml + namespace "nginx-example" deleted + deployment.apps "nginx-deployment" deleted + service "my-nginx" deleted + ``` + + If you want to delete any backups you created, including data in object storage, you can run: `velero backup delete BACKUP_NAME` + + ``` + $ velero backup delete nginx-backup + Are you sure you want to continue (Y/N)? Y + Request to delete backup "nginx-backup" submitted successfully. + The backup will be fully deleted after all associated data (disk snapshots, backup files, restores) are removed. + ``` + + This asks the Velero server to delete all backup data associated with `BACKUP_NAME`. You need to do this for each backup you want to permanently delete. A future version of Velero will allow you to delete multiple backups by name or label selector. + + Once fully removed, the backup is no longer visible when you run: `velero backup get BACKUP_NAME` or more generally `velero backup get`: + + ``` + $ velero backup get nginx-backup + An error occurred: backups.velero.io "nginx-backup" not found + ``` + + ``` + $ velero backup get + NAME STATUS CREATED EXPIRES STORAGE LOCATION SELECTOR + ``` + + + +## Additional Reading + +* [Official Velero Documentation](https://velero.io/docs/master/) +* [Oracle Cloud Infrastructure Documentation](https://docs.cloud.oracle.com/) \ No newline at end of file diff --git a/site/docs/master/support-matrix.md b/site/docs/master/support-matrix.md index e11ed27ea17..6ca796b34c9 100644 --- a/site/docs/master/support-matrix.md +++ b/site/docs/master/support-matrix.md @@ -27,6 +27,7 @@ _Note that these providers are not regularly tested by the Velero team._ * [DigitalOcean][7] * Quobyte * [NooBaa][16] + * [Oracle Cloud][23] _Some storage providers, like Quobyte, may need a different [signature algorithm version][15]._ @@ -71,3 +72,4 @@ After you publish your plugin, open a PR that adds your plugin to the appropriat [20]: https://github.com/openebs/velero-plugin/issues [21]: https://github.com/AliyunContainerService/velero-plugin [22]: https://github.com/AliyunContainerService/velero-plugin/issues +[23]: oracle-config.md diff --git a/site/docs/v1.0.0/oracle-config.md b/site/docs/v1.0.0/oracle-config.md new file mode 100644 index 00000000000..089fa65c40d --- /dev/null +++ b/site/docs/v1.0.0/oracle-config.md @@ -0,0 +1,245 @@ +# Use Oracle Cloud as a Backup Storage Provider for Velero + +## Introduction + +[Velero](https://velero.io/) is a tool used to backup and migrate Kubernetes applications. Here are the steps to use [Oracle Cloud Object Storage](https://docs.cloud.oracle.com/iaas/Content/Object/Concepts/objectstorageoverview.htm) as a destination for Velero backups. + +1. [Download Velero](#download-velero) +2. [Create A Customer Secret Key](#create-a-customer-secret-key) +3. [Create An Oracle Object Storage Bucket](#create-an-oracle-object-storage-bucket) +4. [Install Velero](#install-velero) +5. [Clean Up](#clean-up) +6. [Examples](#examples) +7. [Additional Reading](#additional-reading) + +## Download Velero + +1. Download the [latest release](https://github.com/heptio/velero/releases/) of Velero to your development environment. This includes the `velero` CLI utility and example Kubernetes manifest files. For example: + + ``` + wget https://github.com/heptio/velero/releases/download/v1.0.0/velero-v1.0.0-linux-amd64.tar.gz + ``` + + *We strongly recommend that you use an official release of Velero. The tarballs for each release contain the velero command-line client. The code in the master branch of the Velero repository is under active development and is not guaranteed to be stable!* + +2. Untar the release in your `/usr/bin` directory: `tar -xzvf .tar.gz` + + You may choose to rename the directory `velero` for the sake of simplicty: `mv velero-v1.0.0-linux-amd64 velero` + +3. Add it to your PATH: `export PATH=/usr/local/bin/velero:$PATH` + +4. Run `velero` to confirm the CLI has been installed correctly. You should see an output like this: + +``` +$ velero +Velero is a tool for managing disaster recovery, specifically for Kubernetes +cluster resources. It provides a simple, configurable, and operationally robust +way to back up your application state and associated data. + +If you're familiar with kubectl, Velero supports a similar model, allowing you to +execute commands such as 'velero get backup' and 'velero create schedule'. The same +operations can also be performed as 'velero backup get' and 'velero schedule create'. + +Usage: + velero [command] +``` + + + +## Create A Customer Secret Key + +1. Oracle Object Storage provides an API to enable interoperability with Amazon S3. To use this Amazon S3 Compatibility API, you need to generate the signing key required to authenticate with Amazon S3. This special signing key is an Access Key/Secret Key pair. Follow these steps to [create a Customer Secret Key](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcredentials.htm#To4). Refer to this link for more information about [Working with Customer Secret Keys](https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/managingcredentials.htm#s3). + +2. Create a Velero credentials file with your Customer Secret Key: + + ``` + $ vi credentials-velero + + [default] + aws_access_key_id=bae031188893d1eb83719648790ac850b76c9441 + aws_secret_access_key=MmY9heKrWiNVCSZQ2Mf5XTJ6Ys93Bw2d2D6NMSTXZlk= + ``` + + + +## Create An Oracle Object Storage Bucket + +Create an Oracle Cloud Object Storage bucket called `velero` in the root compartment of your Oracle Cloud tenancy. Refer to this page for [more information about creating a bucket with Object Storage](https://docs.cloud.oracle.com/iaas/Content/Object/Tasks/managingbuckets.htm#usingconsole). + + + +## Install Velero + +You will need the following information to install Velero into your Kubernetes cluster with Oracle Object Storage as the Backup Storage provider: + +``` +velero install \ + --provider [provider name] \ + --bucket [bucket name] \ + --prefix [tenancy name] \ + --use-volume-snapshots=false \ + --secret-file [secret file location] \ + --backup-location-config region=[region],s3ForcePathStyle="true",s3Url=[storage API endpoint] +``` + +- `--provider` Because we are using the S3-compatible API, we will use `aws` as our provider. +- `--bucket` The name of the bucket created in Oracle Object Storage - in our case this is named `velero`. +- ` --prefix` The name of your Oracle Cloud tenancy - in our case this is named `oracle-cloudnative`. +- `--use-volume-snapshots=false` Velero does not currently have a volume snapshot plugin for Oracle Cloud creating volume snapshots is disabled. +- `--secret-file` The path to your `credentials-velero` file. +- `--backup-location-config` The path to your Oracle Object Storage bucket. This consists of your `region` which corresponds to your Oracle Cloud region name ([List of Oracle Cloud Regions](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/regions.htm?Highlight=regions)) and the `s3Url`, the S3-compatible API endpoint for Oracle Object Storage based on your region: `https://oracle-cloudnative.compat.objectstorage.[region name].oraclecloud.com` + +For example: + +``` +velero install \ + --provider aws \ + --bucket velero \ + --prefix oracle-cloudnative \ + --use-volume-snapshots=false \ + --secret-file /Users/mboxell/bin/velero/credentials-velero \ + --backup-location-config region=us-phoenix-1,s3ForcePathStyle="true",s3Url=https://oracle-cloudnative.compat.objectstorage.us-phoenix-1.oraclecloud.com +``` + +This will create a `velero` namespace in your cluster along with a number of CRDs, a ClusterRoleBinding, ServiceAccount, Secret, and Deployment for Velero. If your pod fails to successfully provision, you can troubleshoot your installation by running: `kubectl logs [velero pod name]`. + + + +## Clean Up + +To remove Velero from your environment, delete the namespace, ClusterRoleBinding, ServiceAccount, Secret, and Deployment and delete the CRDs, run: + +``` +kubectl delete namespace/velero clusterrolebinding/velero +kubectl delete crds -l component=velero +``` + +This will remove all resources created by `velero install`. + + + +## Examples + +After creating the Velero server in your cluster, try this example: + +### Basic example (without PersistentVolumes) + +1. Start the sample nginx app: `kubectl apply -f examples/nginx-app/base.yaml` + + This will create an `nginx-example` namespace with a `nginx-deployment` deployment, and `my-nginx` service. + + ``` + $ kubectl apply -f examples/nginx-app/base.yaml + namespace/nginx-example created + deployment.apps/nginx-deployment created + service/my-nginx created + ``` + + You can see the created resources by running `kubectl get all` + + ``` + $ kubectl get all + NAME READY STATUS RESTARTS AGE + pod/nginx-deployment-67594d6bf6-4296p 1/1 Running 0 20s + pod/nginx-deployment-67594d6bf6-f9r5s 1/1 Running 0 20s + + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + service/my-nginx LoadBalancer 10.96.69.166 80:31859/TCP 21s + + NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE + deployment.apps/nginx-deployment 2 2 2 2 21s + + NAME DESIRED CURRENT READY AGE + replicaset.apps/nginx-deployment-67594d6bf6 2 2 2 21s + ``` + +2. Create a backup: `velero backup create nginx-backup --include-namespaces nginx-example` + + ``` + $ velero backup create nginx-backup --include-namespaces nginx-example + Backup request "nginx-backup" submitted successfully. + Run `velero backup describe nginx-backup` or `velero backup logs nginx-backup` for more details. + ``` + + At this point you can navigate to appropriate bucket, which we called `velero`, in the Oracle Cloud Object Storage console to see the resources backed up using Velero. + +3. Simulate a disaster by deleting the `nginx-example` namespace: `kubectl delete namespaces nginx-example` + + ``` + $ kubectl delete namespaces nginx-example + namespace "nginx-example" deleted + ``` + + Wait for the namespace to be deleted. To check that the nginx deployment, service, and namespace are gone, run: + + ``` + kubectl get deployments --namespace=nginx-example + kubectl get services --namespace=nginx-example + kubectl get namespace/nginx-example + ``` + + This should return: `No resources found.` + +4. Restore your lost resources: `velero restore create --from-backup nginx-backup` + + ``` + $ velero restore create --from-backup nginx-backup + Restore request "nginx-backup-20190604102710" submitted successfully. + Run `velero restore describe nginx-backup-20190604102710` or `velero restore logs nginx-backup-20190604102710` for more details. + ``` + + Running `kubectl get namespaces` will show that the `nginx-example` namespace has been restored along with its contents. + +5. Run: `velero restore get` to view the list of restored resources. After the restore finishes, the output looks like the following: + + ``` + $ velero restore get + NAME BACKUP STATUS WARNINGS ERRORS CREATED SELECTOR + nginx-backup-20190604104249 nginx-backup Completed 0 0 2019-06-04 10:42:39 -0700 PDT + ``` + + NOTE: The restore can take a few moments to finish. During this time, the `STATUS` column reads `InProgress`. + + After a successful restore, the `STATUS` column shows `Completed`, and `WARNINGS` and `ERRORS` will show `0`. All objects in the `nginx-example` namespace should be just as they were before you deleted them. + + If there are errors or warnings, for instance if the `STATUS` column displays `FAILED` instead of `InProgress`, you can look at them in detail with `velero restore describe ` + + +6. Clean up the environment with `kubectl delete -f examples/nginx-app/base.yaml` + + ``` + $ kubectl delete -f examples/nginx-app/base.yaml + namespace "nginx-example" deleted + deployment.apps "nginx-deployment" deleted + service "my-nginx" deleted + ``` + + If you want to delete any backups you created, including data in object storage, you can run: `velero backup delete BACKUP_NAME` + + ``` + $ velero backup delete nginx-backup + Are you sure you want to continue (Y/N)? Y + Request to delete backup "nginx-backup" submitted successfully. + The backup will be fully deleted after all associated data (disk snapshots, backup files, restores) are removed. + ``` + + This asks the Velero server to delete all backup data associated with `BACKUP_NAME`. You need to do this for each backup you want to permanently delete. A future version of Velero will allow you to delete multiple backups by name or label selector. + + Once fully removed, the backup is no longer visible when you run: `velero backup get BACKUP_NAME` or more generally `velero backup get`: + + ``` + $ velero backup get nginx-backup + An error occurred: backups.velero.io "nginx-backup" not found + ``` + + ``` + $ velero backup get + NAME STATUS CREATED EXPIRES STORAGE LOCATION SELECTOR + ``` + + + +## Additional Reading + +* [Official Velero Documentation](https://velero.io/docs/master/) +* [Oracle Cloud Infrastructure Documentation](https://docs.cloud.oracle.com/) \ No newline at end of file diff --git a/site/docs/v1.0.0/support-matrix.md b/site/docs/v1.0.0/support-matrix.md index e11ed27ea17..6ca796b34c9 100644 --- a/site/docs/v1.0.0/support-matrix.md +++ b/site/docs/v1.0.0/support-matrix.md @@ -27,6 +27,7 @@ _Note that these providers are not regularly tested by the Velero team._ * [DigitalOcean][7] * Quobyte * [NooBaa][16] + * [Oracle Cloud][23] _Some storage providers, like Quobyte, may need a different [signature algorithm version][15]._ @@ -71,3 +72,4 @@ After you publish your plugin, open a PR that adds your plugin to the appropriat [20]: https://github.com/openebs/velero-plugin/issues [21]: https://github.com/AliyunContainerService/velero-plugin [22]: https://github.com/AliyunContainerService/velero-plugin/issues +[23]: oracle-config.md From 0d326a3903d088d9c37a0d8b1eff002bff75236f Mon Sep 17 00:00:00 2001 From: Steve Kriss Date: Fri, 21 Jun 2019 11:44:13 -0600 Subject: [PATCH 010/118] remove unused hasControllerOwner func and tests Signed-off-by: Steve Kriss --- pkg/restore/restore.go | 12 ------- pkg/restore/restore_test.go | 63 ------------------------------------- 2 files changed, 75 deletions(-) diff --git a/pkg/restore/restore.go b/pkg/restore/restore.go index 0bff67aa6f2..322a3fe7594 100644 --- a/pkg/restore/restore.go +++ b/pkg/restore/restore.go @@ -1113,18 +1113,6 @@ func addRestoreLabels(obj metav1.Object, restoreName, backupName string) { obj.SetLabels(labels) } -// hasControllerOwner returns whether or not an object has a controller -// owner ref. Used to identify whether or not an object should be explicitly -// recreated during a restore. -func hasControllerOwner(refs []metav1.OwnerReference) bool { - for _, ref := range refs { - if ref.Controller != nil && *ref.Controller { - return true - } - } - return false -} - // isCompleted returns whether or not an object is considered completed. // Used to identify whether or not an object should be restored. Only Jobs or Pods are considered func isCompleted(obj *unstructured.Unstructured, groupResource schema.GroupResource) (bool, error) { diff --git a/pkg/restore/restore_test.go b/pkg/restore/restore_test.go index c7ff983465b..5d2db3b07e1 100644 --- a/pkg/restore/restore_test.go +++ b/pkg/restore/restore_test.go @@ -1222,69 +1222,6 @@ func (w *fakeWatch) ResultChan() <-chan watch.Event { return make(chan watch.Event) } -func TestHasControllerOwner(t *testing.T) { - tests := []struct { - name string - object map[string]interface{} - expectOwner bool - }{ - { - name: "missing metadata", - object: map[string]interface{}{}, - }, - { - name: "missing ownerReferences", - object: map[string]interface{}{ - "metadata": map[string]interface{}{}, - }, - expectOwner: false, - }, - { - name: "have ownerReferences, no controller fields", - object: map[string]interface{}{ - "metadata": map[string]interface{}{ - "ownerReferences": []interface{}{ - map[string]interface{}{"foo": "bar"}, - }, - }, - }, - expectOwner: false, - }, - { - name: "have ownerReferences, controller=false", - object: map[string]interface{}{ - "metadata": map[string]interface{}{ - "ownerReferences": []interface{}{ - map[string]interface{}{"controller": false}, - }, - }, - }, - expectOwner: false, - }, - { - name: "have ownerReferences, controller=true", - object: map[string]interface{}{ - "metadata": map[string]interface{}{ - "ownerReferences": []interface{}{ - map[string]interface{}{"controller": false}, - map[string]interface{}{"controller": false}, - map[string]interface{}{"controller": true}, - }, - }, - }, - expectOwner: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - u := &unstructured.Unstructured{Object: test.object} - hasOwner := hasControllerOwner(u.GetOwnerReferences()) - assert.Equal(t, test.expectOwner, hasOwner) - }) - } -} - func TestResetMetadataAndStatus(t *testing.T) { tests := []struct { name string From 8cb9ee9eb8bfba68f482eac191779b6250a65d80 Mon Sep 17 00:00:00 2001 From: jonasrosland Date: Fri, 21 Jun 2019 16:53:00 -0400 Subject: [PATCH 011/118] Add GitHub link in the footer Signed-off-by: jonasrosland --- site/_config.yml | 3 +++ site/_includes/footer.html | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/site/_config.yml b/site/_config.yml index f71994ce3fc..62cc00b4433 100644 --- a/site/_config.yml +++ b/site/_config.yml @@ -31,6 +31,9 @@ footer_social_links: RSS: fa_icon: fa fa-rss url: feed.xml + GitHub: + fa_icon: fab fa-github + url: https://github.com/heptio/velero defaults: - scope: diff --git a/site/_includes/footer.html b/site/_includes/footer.html index 2c22d9cbe2f..cabdd200449 100644 --- a/site/_includes/footer.html +++ b/site/_includes/footer.html @@ -16,7 +16,7 @@
{{ site.footer.title }}