-
Notifications
You must be signed in to change notification settings - Fork 233
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Rework snapshot repositories handling (#752)
### Description This PR reworks how the operator provisions snapshot repositories. The beta feature used a kubernetes job with curl. With this rework the operator does the needed api calls itself using an http client. I've also extracted the logic into its own reconciler for better separation and testing. This reimplementation also removes the limitation of only being able to configure snapshot repositories after cluster provisioning which made it unusable for any kind of GitOps approach. ### Issues Resolved Partially #621 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). Signed-off-by: Sebastian Woehrl <[email protected]>
- Loading branch information
1 parent
0494837
commit 0d0a515
Showing
11 changed files
with
571 additions
and
158 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
6 changes: 6 additions & 0 deletions
6
opensearch-operator/opensearch-gateway/requests/SnapshotRepository.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| package requests | ||
|
|
||
| type SnapshotRepository struct { | ||
| Type string `json:"type"` | ||
| Settings map[string]string `json:"settings,omitempty"` | ||
| } |
5 changes: 5 additions & 0 deletions
5
opensearch-operator/opensearch-gateway/responses/SnapshotRepositoryResponse.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| package responses | ||
|
|
||
| import "github.com/Opster/opensearch-k8s-operator/opensearch-operator/opensearch-gateway/requests" | ||
|
|
||
| type SnapshotRepositoryResponse = map[string]requests.SnapshotRepository |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
113 changes: 113 additions & 0 deletions
113
opensearch-operator/opensearch-gateway/services/os_snapshot_service.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| package services | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| "github.com/Opster/opensearch-k8s-operator/opensearch-operator/opensearch-gateway/requests" | ||
| "github.com/Opster/opensearch-k8s-operator/opensearch-operator/opensearch-gateway/responses" | ||
| "github.com/google/go-cmp/cmp" | ||
| "github.com/google/go-cmp/cmp/cmpopts" | ||
| "github.com/opensearch-project/opensearch-go/opensearchutil" | ||
| "sigs.k8s.io/controller-runtime/pkg/log" | ||
| ) | ||
|
|
||
| var ErrRepoNotFound = errors.New("snapshotRepository not found") | ||
|
|
||
| // checks if the passed SnapshotRepository is same as existing or needs update | ||
| func ShouldUpdateSnapshotRepository(ctx context.Context, newRepository, existingRepository requests.SnapshotRepository) (bool, error) { | ||
| if cmp.Equal(newRepository, existingRepository, cmpopts.EquateEmpty()) { | ||
| return false, nil | ||
| } | ||
| lg := log.FromContext(ctx).WithValues("os_service", "snapshotrepository") | ||
| lg.V(1).Info(fmt.Sprintf("existing SnapshotRepository: %+v", existingRepository)) | ||
| lg.V(1).Info(fmt.Sprintf("new SnapshotRepository: %+v", newRepository)) | ||
| lg.Info("snapshotRepository exists and requires update") | ||
| return true, nil | ||
| } | ||
|
|
||
| // checks if the snapshot repository with the given name already exists | ||
| func SnapshotRepositoryExists(ctx context.Context, service *OsClusterClient, repositoryName string) (bool, error) { | ||
| resp, err := service.GetSnapshotRepository(ctx, repositoryName) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| defer resp.Body.Close() | ||
| if resp.StatusCode == 404 { | ||
| return false, nil | ||
| } else if resp.IsError() { | ||
| return false, fmt.Errorf("response from API is %s", resp.Status()) | ||
| } | ||
| return true, nil | ||
| } | ||
|
|
||
| // fetches the snapshot repository with the given name | ||
| func GetSnapshotRepository(ctx context.Context, service *OsClusterClient, repositoryName string) (*requests.SnapshotRepository, error) { | ||
| resp, err := service.GetSnapshotRepository(ctx, repositoryName) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer resp.Body.Close() | ||
| if resp.StatusCode == 404 { | ||
| return nil, ErrRepoNotFound | ||
| } else if resp.IsError() { | ||
| return nil, fmt.Errorf("response from API is %s", resp.Status()) | ||
| } | ||
| repoResponse := responses.SnapshotRepositoryResponse{} | ||
| if resp != nil && resp.Body != nil { | ||
| err := json.NewDecoder(resp.Body).Decode(&repoResponse) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| // the opensearch api returns a map of name -> repo config, so we extract the one for the repo we need | ||
| repo, exists := repoResponse[repositoryName] | ||
| if !exists { | ||
| return nil, ErrRepoNotFound | ||
| } | ||
| return &repo, nil | ||
| } | ||
| return nil, fmt.Errorf("response is empty") | ||
| } | ||
|
|
||
| // creates the given SnapshotRepository | ||
| func CreateSnapshotRepository(ctx context.Context, service *OsClusterClient, repositoryName string, repository requests.SnapshotRepository) error { | ||
| spec := opensearchutil.NewJSONReader(repository) | ||
| resp, err := service.CreateSnapshotRepository(ctx, repositoryName, spec) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer resp.Body.Close() | ||
| if resp.IsError() { | ||
| return fmt.Errorf("failed to create snapshot repository: %s", resp.String()) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // updates the given SnapshotRepository | ||
| func UpdateSnapshotRepository(ctx context.Context, service *OsClusterClient, repositoryName string, repository requests.SnapshotRepository) error { | ||
| spec := opensearchutil.NewJSONReader(repository) | ||
| resp, err := service.UpdateSnapshotRepository(ctx, repositoryName, spec) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer resp.Body.Close() | ||
| if resp.IsError() { | ||
| return fmt.Errorf("failed to update snapshot repository: %s", resp.String()) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // deletes the given SnapshotRepository | ||
| func DeleteSnapshotRepository(ctx context.Context, service *OsClusterClient, repositoryName string) error { | ||
| resp, err := service.DeleteSnapshotRepository(ctx, repositoryName) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer resp.Body.Close() | ||
| if resp.IsError() { | ||
| return fmt.Errorf("failed to delete snapshot repository: %s", resp.String()) | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.