From 52203f26bdebac9c8c0f0de6642a9c546507449b Mon Sep 17 00:00:00 2001 From: Swapnil Mhamane Date: Fri, 12 Jul 2019 17:55:59 +0530 Subject: [PATCH] Restructure the documentation Signed-off-by: Swapnil Mhamane --- .ci/build | 20 +- .ci/check | 24 +- .ci/integration_test | 21 +- .ci/unit_test | 19 +- Makefile | 4 + README.md | 226 +------------------ build/Dockerfile | 2 +- doc/development/local_setup.md | 76 +++++++ doc/development/testing_and_dependencies.md | 31 +++ doc/{ => proposals}/design.md | 0 doc/{ => proposals}/validation.md | 0 doc/usage/getting_started.md | 89 ++++++++ doc/{ => usage}/metrics.md | 2 +- go.mod | 1 + go.sum | 2 + pkg/snapshot/snapshotter/snapshotter_test.go | 2 +- 16 files changed, 232 insertions(+), 287 deletions(-) create mode 100644 doc/development/local_setup.md create mode 100644 doc/development/testing_and_dependencies.md rename doc/{ => proposals}/design.md (100%) rename doc/{ => proposals}/validation.md (100%) create mode 100644 doc/usage/getting_started.md rename doc/{ => usage}/metrics.md (99%) diff --git a/.ci/build b/.ci/build index 3a9ad3298..69a57d01c 100755 --- a/.ci/build +++ b/.ci/build @@ -34,23 +34,9 @@ VCS="github.com" ORGANIZATION="gardener" PROJECT="etcd-backup-restore" REPOSITORY=${VCS}/${ORGANIZATION}/${PROJECT} +export GO111MODULE=on +cd "${SOURCE_PATH}" -# The `go ` commands requires to see the target repository to be part of a -# Go workspace. Thus, if we are not yet in a Go workspace, let's create one -# temporarily by using symbolic links. -if [[ "${SOURCE_PATH}" != *"src/${REPOSITORY}" ]]; then - SOURCE_SYMLINK_PATH="${SOURCE_PATH}/tmp/src/${REPOSITORY}" - if [[ -d "${SOURCE_PATH}/tmp" ]]; then - rm -rf "${SOURCE_PATH}/tmp" - fi - mkdir -p "${SOURCE_PATH}/tmp/src/${VCS}/${ORGANIZATION}" - ln -s "${SOURCE_PATH}" "${SOURCE_SYMLINK_PATH}" - cd "${SOURCE_SYMLINK_PATH}" - - export GOPATH="${SOURCE_PATH}/tmp" - export GOBIN="${SOURCE_PATH}/tmp/bin" - export PATH="${GOBIN}:${PATH}" -fi ############################################################################### @@ -63,6 +49,7 @@ GIT_SHA=$(git rev-parse --short HEAD || echo "GitNotFound") if [[ -z "$LOCAL_BUILD" ]]; then CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ -a \ + -mod vendor \ -v \ -o ${BINARY_PATH}/linux-amd64/etcdbrctl \ -ldflags "-w -X ${REPOSITORY}/pkg/version.Version=${VERSION} -X ${REPOSITORY}/pkg/version.GitSHA=${GIT_SHA}" \ @@ -72,6 +59,7 @@ if [[ -z "$LOCAL_BUILD" ]]; then else go build \ -v \ + -mod vendor \ -o ${BINARY_PATH}/etcdbrctl \ -ldflags "-w -X ${REPOSITORY}/pkg/version.Version=${VERSION} -X ${REPOSITORY}/pkg/version.GitSHA=${GIT_SHA}" \ main.go diff --git a/.ci/check b/.ci/check index fbc99bc54..81e536a0d 100755 --- a/.ci/check +++ b/.ci/check @@ -27,26 +27,10 @@ VCS="github.com" ORGANIZATION="gardener" PROJECT="etcd-backup-restore" REPOSITORY=${VCS}/${ORGANIZATION}/${PROJECT} - -# The `go ` commands requires to see the target repository to be part of a -# Go workspace. Thus, if we are not yet in a Go workspace, let's create one -# temporarily by using symbolic links. -if [[ "${SOURCE_PATH}" != *"src/${REPOSITORY}" ]]; then - SOURCE_SYMLINK_PATH="${SOURCE_PATH}/tmp/src/${REPOSITORY}" - if [[ -d "${SOURCE_PATH}/tmp" ]]; then - rm -rf "${SOURCE_PATH}/tmp" - fi - mkdir -p "${SOURCE_PATH}/tmp/src/${VCS}/${ORGANIZATION}" - ln -s "${SOURCE_PATH}" "${SOURCE_SYMLINK_PATH}" - cd "${SOURCE_SYMLINK_PATH}" - - export GOPATH="${SOURCE_PATH}/tmp" - export GOBIN="${SOURCE_PATH}/tmp/bin" - export PATH="${GOBIN}:${PATH}" -fi +cd "${SOURCE_PATH}" # Install Golint (linting tool). -go get -u golang.org/x/lint/golint +GO111MODULE=off go get -u golang.org/x/lint/golint # Install Helm from binary. curl -LO https://git.io/get_helm.sh @@ -56,12 +40,12 @@ rm get_helm.sh ############################################################################### -PACKAGES="$(go list -e ./... | grep -vE '/tmp/|/vendor/')" +PACKAGES="$(GO111MODULE=on go list -mod=vendor -e ./...)" LINT_FOLDERS="$(echo ${PACKAGES} | sed "s|github.com/gardener/etcd-backup-restore|.|g")" HELM_CHART_PATH="${SOURCE_PATH}/chart/etcd-backup-restore" # Execute static code checks. -go vet ${PACKAGES} +GO111MODULE=on go vet -mod vendor ${PACKAGES} # Execute automatic code formatting directive. go fmt ${PACKAGES} diff --git a/.ci/integration_test b/.ci/integration_test index 595ed09c9..f899a7747 100755 --- a/.ci/integration_test +++ b/.ci/integration_test @@ -30,22 +30,9 @@ REPOSITORY=${VCS}/${ORGANIZATION}/${PROJECT} VERSION_FILE="$(readlink -f "${SOURCE_PATH}/VERSION")" VERSION="$(cat "${VERSION_FILE}")" -# The `go ` commands requires to see the target repository to be part of a -# Go workspace. Thus, if we are not yet in a Go workspace, let's create one -# temporarily by using symbolic links. -if [[ "${SOURCE_PATH}" != *"src/${REPOSITORY}" ]]; then - SOURCE_SYMLINK_PATH="${SOURCE_PATH}/tmp/src/${REPOSITORY}" - if [[ -d "${SOURCE_PATH}/tmp" ]]; then - rm -rf "${SOURCE_PATH}/tmp" - fi - mkdir -p "${SOURCE_PATH}/tmp/src/${VCS}/${ORGANIZATION}" - ln -s "${SOURCE_PATH}" "${SOURCE_SYMLINK_PATH}" - cd "${SOURCE_SYMLINK_PATH}" - - export GOPATH="${SOURCE_PATH}/tmp" - export GOBIN="${SOURCE_PATH}/tmp/bin" - export PATH="${GOBIN}:${PATH}" -fi +export GOBIN="${SOURCE_PATH}/bin" +export PATH="${GOBIN}:${PATH}" +cd "${SOURCE_PATH}" ############################################################################## @@ -64,7 +51,7 @@ function setup_test_enviornment() { function setup_ginkgo(){ echo "Installing Ginkgo..." - go get -u github.com/onsi/ginkgo/ginkgo + GO111MODULE=off go get -u github.com/onsi/ginkgo/ginkgo echo "Successfully installed Ginkgo." } diff --git a/.ci/unit_test b/.ci/unit_test index 938995771..33f9e0cf1 100755 --- a/.ci/unit_test +++ b/.ci/unit_test @@ -27,22 +27,7 @@ ORGANIZATION="gardener" PROJECT="etcd-backup-restore" REPOSITORY=${VCS}/${ORGANIZATION}/${PROJECT} -# The `go ` commands requires to see the target repository to be part of a -# Go workspace. Thus, if we are not yet in a Go workspace, let's create one -# temporarily by using symbolic links. -if [[ "${SOURCE_PATH}" != *"src/${REPOSITORY}" ]]; then - SOURCE_SYMLINK_PATH="${SOURCE_PATH}/tmp/src/${REPOSITORY}" - if [[ -d "${SOURCE_PATH}/tmp" ]]; then - rm -rf "${SOURCE_PATH}/tmp" - fi - mkdir -p "${SOURCE_PATH}/tmp/src/${VCS}/${ORGANIZATION}" - ln -s "${SOURCE_PATH}" "${SOURCE_SYMLINK_PATH}" - cd "${SOURCE_SYMLINK_PATH}" - - export GOPATH="${SOURCE_PATH}/tmp" - export GOBIN="${SOURCE_PATH}/tmp/bin" - export PATH="${GOBIN}:${PATH}" -fi +cd "${SOURCE_PATH}" # Install Ginkgo (test framework) to be able to execute the tests. go get -u github.com/onsi/ginkgo/ginkgo @@ -57,7 +42,7 @@ function test_with_coverage() { ginkgo $GINKGO_COMMON_FLAGS --coverprofile ${coverprofile_file} -covermode=set -outputdir ${output_dir} ${TEST_PACKAGES} sed -i '/mode: set/d' ${output_dir}/${coverprofile_file} {( echo "mode: set"; cat ${output_dir}/${coverprofile_file} )} > ${output_dir}/${coverprofile_file}.temp - mv ${output_dir}/${coverprofile_file}.temp ${output_dir}/${coverprofile_file} + mv ${output_dir}/${coverprofile_file}.temp ${output_dir}/${coverprofile_file} go tool cover -func ${output_dir}/${coverprofile_file} } diff --git a/Makefile b/Makefile index fefb39159..6bd4e1a5a 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,10 @@ revendor: @env GO111MODULE=on go mod vendor -v @env GO111MODULE=on go mod tidy -v +.PHONY: update-dependencies +update-dependencies: + @env GO111MODULE=on go get -u + .PHONY: build build: @.ci/build diff --git a/README.md b/README.md index 132e35fba..8bc2ca93b 100644 --- a/README.md +++ b/README.md @@ -6,225 +6,23 @@ Etcd-backup-restore is collection of components to backup and restore the [etcd]. It also, provides the ability to validate the data directory, so that we could know the data directory is in good shape to bootstrap etcd successfully. -## Table of Contents +## Documentation Index -- [Getting started](#getting-started) - - [Prerequisites](#prerequisites) - - [Build](#build) -- [Design](#design) -- [Usage](#usage) - - [Cloud Provider Credentials](#cloud-provider-credentials) - - [Taking scheduled snapshot](#taking-scheduled-snapshot) - - [Etcd data directory initialization](#etcd-data-directory-initialization) - - [Etcdbrctl server](#etcdbrctl-server) -- [Dependency management](#dependency-management) - - [Updating dependencies](#updating-dependencies) -- [Testing](#testing) +### Usage -## Getting started +* [Getting started](doc/usage/getting_started.md) +* [Monitoring](doc/usage/metrics.md) -Currently there are no binary build available, but it is pretty straight forward to build it by following the steps mentioned below. +### Design and Proposals +* [Core design](doc/proposals/design.md) +* [Etcd data Validation ](doc/proposals/validation.md) +* [High watch events ingress rate issue](doc.proposals/high_watch_event_ingress_rate.md) -### Prerequisites +### Development -Although the following installation instructions are for Mac OS X, similar alternate commands could be found for any Linux distribution +* [Setting up a local development environment](doc.development/local_setup.md) +* [Testing and Dependency Management](doc.development/testing_and_dependencies.md) +* [Adding support for a new cloud provider](doc/development/new_cp_support.md) -#### Installing [Golang](https://golang.org/) environment - -Install the latest version of Golang (at least `v1.9.4` is required). For Mac OS, you could use [Homebrew](https://brew.sh/): - -```sh -brew install golang -``` - -For other OS, please check [Go installation documentation](https://golang.org/doc/install). - -Make sure to set your `$GOPATH` environment variable properly (conventionally, it points to `$HOME/go`). - -For your convenience, you can add the `bin` directory of the `$GOPATH` to your `$PATH`: `PATH=$PATH:$GOPATH/bin`, but it is not necessarily required. - -We use [Dep](https://github.com/golang/dep) for managing golang package dependencies. Please install it -on Mac OS via - -```sh -brew install dep -``` - -On other operating systems, please check the [Dep installation documentation](https://golang.github.io/dep/docs/installation.html) and the [Dep releases page](https://github.com/golang/dep/releases). After downloading the appropriate release in your `$GOPATH/bin` folder, you need to make it executable via `chmod +x ` and rename it to dep via `mv dep- dep`. - -#### [Golint](https://github.com/golang/lint) - -In order to perform linting on the Go source code, please install [Golint](https://github.com/golang/lint): - -```bash -go get -u github.com/golang/lint/golint -``` - -#### [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) - -In order to perform tests on the Go source code, please install [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](http://onsi.github.io/gomega/). Please make yourself familiar with both frameworks and read their introductions after installation: - -```bash -go get -u github.com/onsi/ginkgo/ginkgo -go get -u github.com/onsi/gomega -``` - -#### Installing `git` - -We use `git` as VCS which you would need to install. - -On Mac OS run - -```sh -brew install git -``` - -#### Installing `gcloud` SDK (Optional) - -In case you have to create a new release or a new hotfix, you have to push the resulting Docker image into a Docker registry. Currently, we are using the Google Container Registry (this could change in the future). Please follow the official [installation instructions from Google](https://cloud.google.com/sdk/downloads). - -### Installing `Docker` (Optional) - -In case you want to build Docker images, you have to install Docker itself. We recommend using [Docker for Mac OS X](https://docs.docker.com/docker-for-mac/) which can be downloaded from [here](https://download.docker.com/mac/stable/Docker.dmg). - -### Build - -First, you need to create a target folder structure before cloning and building `etcdbrctl`. - -```sh - -mkdir -p ~/go/src/github.com/gardener -cd ~/go/src/github.com/gardener -git clone https://github.com/gardener/etcd-backup-restore.git -cd etcd-backup-restore -``` - -To build the binary in local machine environment, use `make` target `build-local`. - -```sh -make build-local -``` - -This will build the binary `etcdbrctl` under `bin` directory. - -Next you can make it available to use as shell command by moving the executable to `/usr/local/bin`. - -## Design - -Please find the design doc [here](doc/design.md). - -## Usage - -You can follow the `help` flag on `etcdbrctl` command and its sub-commands to know the usage details. Some of the common use cases are mentioned below. Although examples below uses AWS S3 as storage provider, we have added support for AWS, GCS, Azure, Openstack swift and Alicloud OSS object store. It also supports local disk as storage provider. - -### Cloud Provider Credentials - -The procedure to provide credentials to access the cloud provider object store varies for different providers. - -For `AWS S3`, the `credentials` file has to be provided in the `~/.aws` directory. - -For `GCP Containers`, the service account json file should be provided in the `~/.gcp` as a `service-account-file.json` file. - -For `Azure Blob storage`, `STORAGE_ACCOUNT` and `STORAGE_KEY` should be made available as environment variables. - -For `Openstack Swift`, `OS_USERNAME`, `OS_PASSWORD`, `OS_AUTH_URL`, `OS_TENANT_ID` and `OS_DOMAIN_ID` should be made available as environment variables. - -For `Alicloud OSS`, `ALICLOUD_ENDPOINT`, `ALICLOUD_ACCESS_KEY_ID`, `ALICLOUD_ACCESS_KEY_SECRET` should be made available as environment variables. - -### Taking scheduled snapshot - - - -`etcd` should already be running. One can apply standard cron format scheduling for regular backup of etcd. The cron schedule is used to take full backups. The delta snapshots are taken at regular intervals in the period in between full snapshots as indicated by the `delta-snapshot-period-seconds` flag. The default for the same is 10 seconds. - -etcd-backup-restore has two garbage collection policies to collect existing backups from the cloud bucket. The flag `garbage-collection-policy` is used to indicate the correct garbage collection policy. -1. `Exponential` -1. `LimitBased` - -If using `LimitBased` policy, the `max-backups` flag should be provided to indicate the number of recent backups to persist at each garbage collection cycle. - -```sh -$ ./bin/etcdbrctl snapshot --storage-provider="S3" --etcd-endpoints http://localhost:2379 --schedule "*/1 * * * *" --store-container="etcd-backup" --delta-snapshot-period-seconds=10 --max-backups=10 --garbage-collection-policy='LimitBased' -INFO[0000] Validating schedule... -INFO[0000] Job attempt: 1 -INFO[0000] Taking initial full snapshot at time: 2018-07-09 12:09:04.3567024 +0000 UTC -INFO[0000] Successfully opened snapshot reader on etcd -INFO[0000] Successfully saved full snapshot at: Backup-1531138145/Full-00000000-00000001-1531138145 -INFO[0000] Will take next full snapshot at time: 2018-07-09 12:10:00 +0000 UTC -INFO[0000] Applied watch on etcd from revision: 00000002 -INFO[0000] No events received to save snapshot. -``` - -The command mentioned above takes hourly snapshots and pushs it to S3 bucket named "etcd-backup". It is configured to keep only last 10 backups in bucket. - -`Exponential` policy stores the snapshots in a condensed manner as mentioned below: -- All full backups and delta backups for the previous hour. -- Latest full snapshot of each previous hour for the day. -- Latest full snapshot of each previous day for 7 days. -- Latest full snapshot of the previous 4 weeks. - -```sh -$ ./bin/etcdbrctl snapshot --storage-provider="S3" --etcd-endpoints http://localhost:2379 --schedule "*/1 * * * *" --store-container="etcd-backup" --delta-snapshot-period-seconds=10 --garbage-collection-policy='Exponential' -INFO[0000] Validating schedule... -INFO[0000] Job attempt: 1 -INFO[0000] Taking initial full snapshot at time: 2018-07-09 12:09:04.3567024 +0000 UTC -INFO[0000] Successfully opened snapshot reader on etcd -INFO[0000] Successfully saved full snapshot at: Backup-1531138145/Full-00000000-00000001-1531138145 -INFO[0000] Will take next full snapshot at time: 2018-07-09 12:10:00 +0000 UTC -INFO[0000] Applied watch on etcd from revision: 00000002 -INFO[0000] No events received to save snapshot. -``` - -The command mentioned above stores etcd snapshots as per the exponential policy mentioned above. - -### Etcd data directory initialization - -Sub-command `initialize` does the task of data directory validation. If the data directory is found to be corrupt, the controller will restore it from the latest snapshot in the cloud store. It restores the full snapshot first and then incrementally applies the delta snapshots. - -```sh -$ ./bin/etcdbrctl initialize --storage-provider="S3" --store-container="etcd-backup" --data-dir="default.etcd" -INFO[0000] Checking for data directory structure validity... -INFO[0000] Checking for data directory files corruption... -INFO[0000] Verifying snap directory... -Verifying Snapfile default.etcd/member/snap/0000000000000001-0000000000000001.snap. -INFO[0000] Verifying WAL directory... -INFO[0000] Verifying DB file... -INFO[0000] Data directory corrupt. Invalid db files: invalid database -INFO[0000] Removing data directory(default.etcd) for snapshot restoration. -INFO[0000] Finding latest snapshot... -INFO[0000] Restoring from latest snapshot: Full-00000000-00040010-1522152360... -2018-03-27 17:38:06.617280 I | etcdserver/membership: added member 8e9e05c52164694d [http://localhost:2380] to cluster cdf818194e3a8c32 -INFO[0000] Successfully restored the etcd data directory. -``` - -### Etcdbrctl server - -With sub-command `server` you can start a http server which exposes an endpoint to initialize etcd over REST interface. The server also keeps on backup schedule thread running to have periodic backups. This is mainly made available to manage an etcd instance running in a Kubernetes cluster. You can deploy the example [manifest](./example/etcd-statefulset.yaml) on a Kubernetes cluster to have an fault resilient etcd instance. - -## Dependency management - -We use [Dep](https://github.com/golang/dep) to manage golang dependencies.. In order to add a new package dependency to the project, you can perform `dep ensure -add ` or edit the `Gopkg.toml` file and append the package along with the version you want to use as a new `[[constraint]]`. - -### Updating dependencies - -The `Makefile` contains a rule called `revendor` which performs a `dep ensure -update` and a `dep prune` command. This updates all the dependencies to its latest versions (respecting the constraints specified in the `Gopkg.toml` file). The command also installs the packages which do not already exist in the `vendor` folder but are specified in the `Gopkg.toml` (in case you have added new ones). - -```sh -make revendor -``` - -The dependencies are installed into the `vendor` folder which **should be added** to the VCS. - -:warning: Make sure you test the code after you have updated the dependencies! - -## Testing - -We have created `make` target `verify` which will internally run different rule like `fmt` for formatting, `lint` for linting check and most importantly `test` which will check the code against predefined unit tests. Although, currently there are not enough test cases written to cover entire code, hence one should check for failure cases manually before raising pull request. We will eventually add the test cases for complete code coverage. - -```sh -make verify -``` - -By default, we try to run test in parallel without computing code coverage. To get the code coverage, you will have to set environment variable `COVER` to `true`. This will log the code coverage percentage at the end of test logs. Also, all cover profile files will accumulated under `test/output/coverprofile.out` directory. You can visualize exact code coverage using `make show-coverage`. [etcd]: https://github.com/coreos/etcd diff --git a/build/Dockerfile b/build/Dockerfile index 8ce5874ac..3bdf500c4 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM alpine:3.9.3 +FROM alpine:3.10.1 RUN apk add --update bash curl diff --git a/doc/development/local_setup.md b/doc/development/local_setup.md new file mode 100644 index 000000000..8caa5cc54 --- /dev/null +++ b/doc/development/local_setup.md @@ -0,0 +1,76 @@ +## Prerequisites + +Although the following installation instructions are for Mac OS X, similar alternate commands could be found for any Linux distribution + +### Installing [Golang](https://golang.org/) environment + +Install the latest version of Golang (at least `v1.12` is required). For Mac OS, you could use [Homebrew](https://brew.sh/): + +```sh +brew install golang +``` + +For other OS, please check [Go installation documentation](https://golang.org/doc/install). + +Make sure to set your `$GOPATH` environment variable properly (conventionally, it points to `$HOME/go`). + +For your convenience, you can add the `bin` directory of the `$GOPATH` to your `$PATH`: `PATH=$PATH:$GOPATH/bin`, but it is not necessarily required. + +### [Golint](https://github.com/golang/lint) + +In order to perform linting on the Go source code, please install [Golint](https://github.com/golang/lint): + +```bash +go get -u golang.org/x/lint/golint +``` + +### [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) + +In order to perform tests on the Go source code, please install [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](http://onsi.github.io/gomega/). Please make yourself familiar with both frameworks and read their introductions after installation: + +```bash +go get -u github.com/onsi/ginkgo/ginkgo +go get -u github.com/onsi/gomega +``` + +### Installing `git` + +We use `git` as VCS which you would need to install. + +On Mac OS run + +```sh +brew install git +``` + +### Installing `gcloud` SDK (Optional) + +In case you have to create a new release or a new hotfix, you have to push the resulting Docker image into a Docker registry. Currently, we are using the Google Container Registry (this could change in the future). Please follow the official [installation instructions from Google](https://cloud.google.com/sdk/downloads). + + +## Build + +Currently there are no binary build available, but it is pretty straight forward to build it by following the steps mentioned below. + +* First, you need to create a target folder structure before cloning and building `etcdbrctl`. + + ```sh + git clone https://github.com/gardener/etcd-backup-restore.git + cd etcd-backup-restore + ``` + +* To build the binary in local machine environment, use `make` target `build-local`. It will build the binary `etcdbrctl` under `bin` directory. + + ```sh + make build-local + ``` + +* Next you can make it available to use as shell command by moving the executable to `/usr/local/bin`. +You can verify the installation by running following command: + ```console + $ etcdbrctl -v + INFO[0000] etcd-backup-restore Version: v0.7.0-dev + INFO[0000] Git SHA: 38979f0 + INFO[0000] Go Version: go1.12 + INFO[0000] Go OS/Arch: darwin/amd64 + ``` \ No newline at end of file diff --git a/doc/development/testing_and_dependencies.md b/doc/development/testing_and_dependencies.md new file mode 100644 index 000000000..aca981665 --- /dev/null +++ b/doc/development/testing_and_dependencies.md @@ -0,0 +1,31 @@ +# Dependency management + +We use golang modules to manage golang dependencies. In order to add a new package dependency to the project, you can perform `go get @` or edit the `go.mod` file and append the package along with the version you want to use. + +### Updating dependencies + +The `Makefile` contains a rule called `revendor` which performs `go mod vendor` and `go mod tidy`. +* `go mod vendor` resets the main module's vendor directory to include all packages needed to build and test all the main module's packages. It does not include test code for vendored packages. +It does not include test code for vendored packages. +* `go mod tidy` makes sure go.mod matches the source code in the module. +It adds any missing modules necessary to build the current module's +packages and dependencies, and it removes unused modules that +don't provide any relevant packages. + +```sh +make revendor +``` + +The dependencies are installed into the `vendor` folder which **should be added** to the VCS. + +:warning: Make sure you test the code after you have updated the dependencies! + +# Testing + +We have created `make` target `verify` which will internally run different rule like `fmt` for formatting, `lint` for linting check and most importantly `test` which will check the code against predefined unit tests. Although, currently there are not enough test cases written to cover entire code, hence one should check for failure cases manually before raising pull request. We will eventually add the test cases for complete code coverage. + +```sh +make verify +``` + +By default, we try to run test in parallel without computing code coverage. To get the code coverage, you will have to set environment variable `COVER` to `true`. This will log the code coverage percentage at the end of test logs. Also, all cover profile files will accumulated under `test/output/coverprofile.out` directory. You can visualize exact code coverage using `make show-coverage`. diff --git a/doc/design.md b/doc/proposals/design.md similarity index 100% rename from doc/design.md rename to doc/proposals/design.md diff --git a/doc/validation.md b/doc/proposals/validation.md similarity index 100% rename from doc/validation.md rename to doc/proposals/validation.md diff --git a/doc/usage/getting_started.md b/doc/usage/getting_started.md new file mode 100644 index 000000000..cead71c71 --- /dev/null +++ b/doc/usage/getting_started.md @@ -0,0 +1,89 @@ +# Getting started + +Currently we don't publish the binary build with the release, but it is pretty straight forward to build it by following the steps mentioned [here](../development/local_setup.md#build). But we do publish the docker image with each release, please check the [release page](https://github.com/gardener/etcd-backup-restore/releases) for the same. + +## Usage + +You can follow the `help` flag on `etcdbrctl` command and its sub-commands to know the usage details. Some of the common use cases are mentioned below. Although examples below uses AWS S3 as storage provider, we have added support for AWS, GCS, Azure, Openstack swift and Alicloud OSS object store. It also supports local disk as storage provider. + +### Cloud Provider Credentials + +The procedure to provide credentials to access the cloud provider object store varies for different providers. + +* For `AWS S3`, the `credentials` file has to be provided in the `~/.aws` directory. + +* For `GCP Containers`, the service account json file should be provided in the `~/.gcp` as a `service-account-file.json` file. + +* For `Azure Blob storage`, `STORAGE_ACCOUNT` and `STORAGE_KEY` should be made available as environment variables. + +* For `Openstack Swift`, `OS_USERNAME`, `OS_PASSWORD`, `OS_AUTH_URL`, `OS_TENANT_ID` and `OS_DOMAIN_ID` should be made available as environment variables. + +* For `Alicloud OSS`, `ALICLOUD_ENDPOINT`, `ALICLOUD_ACCESS_KEY_ID`, `ALICLOUD_ACCESS_KEY_SECRET` should be made available as environment variables. + +### Taking scheduled snapshot + +`etcd` should already be running. One can apply standard cron format scheduling for regular backup of etcd. The cron schedule is used to take full backups. The delta snapshots are taken at regular intervals in the period in between full snapshots as indicated by the `delta-snapshot-period-seconds` flag. The default for the same is 10 seconds. + +etcd-backup-restore has two garbage collection policies to collect existing backups from the cloud bucket. The flag `garbage-collection-policy` is used to indicate the correct garbage collection policy. +1. `Exponential` +1. `LimitBased` + +If using `LimitBased` policy, the `max-backups` flag should be provided to indicate the number of recent backups to persist at each garbage collection cycle. + +```console +$ ./bin/etcdbrctl snapshot --storage-provider="S3" --etcd-endpoints http://localhost:2379 --schedule "*/1 * * * *" --store-container="etcd-backup" --delta-snapshot-period-seconds=10 --max-backups=10 --garbage-collection-policy='LimitBased' +INFO[0000] Validating schedule... +INFO[0000] Job attempt: 1 +INFO[0000] Taking initial full snapshot at time: 2018-07-09 12:09:04.3567024 +0000 UTC +INFO[0000] Successfully opened snapshot reader on etcd +INFO[0000] Successfully saved full snapshot at: Backup-1531138145/Full-00000000-00000001-1531138145 +INFO[0000] Will take next full snapshot at time: 2018-07-09 12:10:00 +0000 UTC +INFO[0000] Applied watch on etcd from revision: 00000002 +INFO[0000] No events received to save snapshot. +``` + +The command mentioned above takes hourly snapshots and pushs it to S3 bucket named "etcd-backup". It is configured to keep only last 10 backups in bucket. + +`Exponential` policy stores the snapshots in a condensed manner as mentioned below: +- All full backups and delta backups for the previous hour. +- Latest full snapshot of each previous hour for the day. +- Latest full snapshot of each previous day for 7 days. +- Latest full snapshot of the previous 4 weeks. + +```console +$ ./bin/etcdbrctl snapshot --storage-provider="S3" --etcd-endpoints http://localhost:2379 --schedule "*/1 * * * *" --store-container="etcd-backup" --delta-snapshot-period-seconds=10 --garbage-collection-policy='Exponential' +INFO[0000] Validating schedule... +INFO[0000] Job attempt: 1 +INFO[0000] Taking initial full snapshot at time: 2018-07-09 12:09:04.3567024 +0000 UTC +INFO[0000] Successfully opened snapshot reader on etcd +INFO[0000] Successfully saved full snapshot at: Backup-1531138145/Full-00000000-00000001-1531138145 +INFO[0000] Will take next full snapshot at time: 2018-07-09 12:10:00 +0000 UTC +INFO[0000] Applied watch on etcd from revision: 00000002 +INFO[0000] No events received to save snapshot. +``` + +The command mentioned above stores etcd snapshots as per the exponential policy mentioned above. + +### Etcd data directory initialization + +Sub-command `initialize` does the task of data directory validation. If the data directory is found to be corrupt, the controller will restore it from the latest snapshot in the cloud store. It restores the full snapshot first and then incrementally applies the delta snapshots. + +```console +$ ./bin/etcdbrctl initialize --storage-provider="S3" --store-container="etcd-backup" --data-dir="default.etcd" +INFO[0000] Checking for data directory structure validity... +INFO[0000] Checking for data directory files corruption... +INFO[0000] Verifying snap directory... +Verifying Snapfile default.etcd/member/snap/0000000000000001-0000000000000001.snap. +INFO[0000] Verifying WAL directory... +INFO[0000] Verifying DB file... +INFO[0000] Data directory corrupt. Invalid db files: invalid database +INFO[0000] Removing data directory(default.etcd) for snapshot restoration. +INFO[0000] Finding latest snapshot... +INFO[0000] Restoring from latest snapshot: Full-00000000-00040010-1522152360... +2018-03-27 17:38:06.617280 I | etcdserver/membership: added member 8e9e05c52164694d [http://localhost:2380] to cluster cdf818194e3a8c32 +INFO[0000] Successfully restored the etcd data directory. +``` + +### Etcdbrctl server + +With sub-command `server` you can start a http server which exposes an endpoint to initialize etcd over REST interface. The server also keeps on backup schedule thread running to have periodic backups. This is mainly made available to manage an etcd instance running in a Kubernetes cluster. You can deploy the example [helm chart](../../chart/etcd-backup-restore) on a Kubernetes cluster to have an fault resilient etcd instance. diff --git a/doc/metrics.md b/doc/usage/metrics.md similarity index 99% rename from doc/metrics.md rename to doc/usage/metrics.md index a85a12350..b83c2b3f3 100644 --- a/doc/metrics.md +++ b/doc/usage/metrics.md @@ -1,4 +1,4 @@ -# Metrics +# Monitoring etcd-backup-restore uses [Prometheus][prometheus] for metrics reporting. The metrics can be used for real-time monitoring and debugging. It won't persist its metrics; if a member restarts, the metrics will be reset. diff --git a/go.mod b/go.mod index 0e5741412..e90c3bd5c 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/aws/aws-sdk-go v1.19.23 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/beorn7/perks v1.0.0 // indirect + github.com/boltdb/bolt v1.3.1 github.com/coreos/bbolt v1.3.2 github.com/coreos/etcd v3.3.13+incompatible github.com/coreos/go-semver v0.3.0 // indirect diff --git a/go.sum b/go.sum index e246f265d..f254baca0 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= diff --git a/pkg/snapshot/snapshotter/snapshotter_test.go b/pkg/snapshot/snapshotter/snapshotter_test.go index d59e51510..ec6e24ce1 100644 --- a/pkg/snapshot/snapshotter/snapshotter_test.go +++ b/pkg/snapshot/snapshotter/snapshotter_test.go @@ -44,7 +44,7 @@ var _ = Describe("Snapshotter", func() { insecureTransport bool insecureSkipVerify bool etcdUsername string - etcdPassword string + etcdPassword string err error ) BeforeEach(func() {