forked from runatlantis/atlantis
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement local plan storage. Refactor S3 storage (runatlantis#33)
* added new flags `plan-backend`, `plan-s3-bucket`, `plan-s3-prefix` and deleted `s3-bucket` * interface `plan.Backend` that is implemented by `file` and `s3` * simplified s3 code * didn't end up following my suggestions in runatlantis#30 since storing stuff in metadata requires you to `Get` the object *first* and then use the metadata. By parsing the `Key` to get repo, pull, path, and env, it skips an initial `Get` step, and I can download directly to the correct directory * allow users to specify `application/json` or `application/x-www-form-urlencoded` for the webhook delivery type * remove sending of special header for pull request api (fixes runatlantis#11) Closes runatlantis#30 and runatlantis#17 and runatlantis#11
- Loading branch information
Showing
19 changed files
with
577 additions
and
511 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| package plan | ||
|
|
||
| import ( | ||
| "github.com/hootsuite/atlantis/models" | ||
| ) | ||
|
|
||
| type Backend interface { | ||
| SavePlan(path string, project models.Project, env string, pullNum int) error | ||
| CopyPlans(dstRepoPath string, repoFullName string, env string, pullNum int) ([]Plan, error) | ||
| } | ||
|
|
||
| type Plan struct { | ||
| Project models.Project | ||
| // LocalPath is the path to the plan on disk | ||
| LocalPath string | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| package file | ||
|
|
||
| import ( | ||
| "github.com/hootsuite/atlantis/models" | ||
| "github.com/hootsuite/atlantis/plan" | ||
| "github.com/pkg/errors" | ||
| "io/ioutil" | ||
| "os" | ||
| "path/filepath" | ||
| "strconv" | ||
| ) | ||
|
|
||
| type Backend struct { | ||
| // baseDir is the root at which all plans will be stored | ||
| baseDir string | ||
| } | ||
|
|
||
| func New(baseDir string) (*Backend, error) { | ||
| baseDir = filepath.Clean(baseDir) | ||
| if err := os.MkdirAll(baseDir, 0755); err != nil { | ||
| return nil, err | ||
| } | ||
| return &Backend{baseDir}, nil | ||
| } | ||
|
|
||
| // save plans to baseDir/owner/repo/pullNum/path/env.tfplan | ||
| func (b *Backend) SavePlan(path string, project models.Project, env string, pullNum int) error { | ||
| savePath := b.path(project, pullNum) | ||
| if err := os.MkdirAll(savePath, 0755); err != nil { | ||
| return errors.Wrap(err, "creating save directory") | ||
| } | ||
| if err := b.copy(path, filepath.Join(savePath, env+".tfplan")); err != nil { | ||
| return errors.Wrap(err, "saving plan") | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (b *Backend) CopyPlans(dstRepo string, repoFullName string, env string, pullNum int) ([]plan.Plan, error) { | ||
| // Look in the directory for this repo/pull and get plans for all projects. | ||
| // Then filter to the plans for this environment | ||
| var toCopy []string // will contain paths to the plan files relative to repo root | ||
| root := filepath.Join(b.baseDir, repoFullName, strconv.Itoa(pullNum)) | ||
| err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { | ||
| if err != nil { | ||
| return err | ||
| } | ||
| // if the plan is for the right env, | ||
| if info.Name() == env+".tfplan" { | ||
| rel, err := filepath.Rel(root, path) | ||
| if err == nil { | ||
| toCopy = append(toCopy, rel) | ||
| } | ||
| } | ||
| return nil | ||
| }) | ||
|
|
||
| var plans []plan.Plan | ||
| if err != nil { | ||
| return plans, errors.Wrap(err, "listing plans") | ||
| } | ||
|
|
||
| // copy the plans to the destination repo | ||
| for _, file := range toCopy { | ||
| dst := filepath.Join(dstRepo, file) | ||
| if err := b.copy(filepath.Join(root, file), dst); err != nil { | ||
| return plans, errors.Wrap(err, "copying plan") | ||
| } | ||
| plans = append(plans, plan.Plan{ | ||
| Project: models.Project{ | ||
| Path: filepath.Dir(file), | ||
| RepoFullName: repoFullName, | ||
| }, | ||
| LocalPath: dst, | ||
| }) | ||
| } | ||
| return plans, nil | ||
| } | ||
|
|
||
| func (b *Backend) copy(src string, dst string) error { | ||
| data, err := ioutil.ReadFile(src) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "reading %s", src) | ||
| } | ||
|
|
||
| if err = ioutil.WriteFile(dst, data, 0644); err != nil { | ||
| return errors.Wrapf(err, "writing %s", dst) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (b *Backend) path(p models.Project, pullNum int) string { | ||
| return filepath.Join(b.baseDir, p.RepoFullName, strconv.Itoa(pullNum), p.Path) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| package s3 | ||
|
|
||
| import ( | ||
| "os" | ||
| pathutil "path" | ||
| "path/filepath" | ||
| "strconv" | ||
|
|
||
| "github.com/aws/aws-sdk-go/aws" | ||
| "github.com/aws/aws-sdk-go/aws/client" | ||
| "github.com/aws/aws-sdk-go/service/s3" | ||
| "github.com/aws/aws-sdk-go/service/s3/s3manager" | ||
| "github.com/hootsuite/atlantis/models" | ||
| "github.com/hootsuite/atlantis/plan" | ||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| type Backend struct { | ||
| s3 *s3.S3 | ||
| uploader *s3manager.Uploader | ||
| downloader *s3manager.Downloader | ||
| bucket string | ||
| keyPrefix string | ||
| } | ||
|
|
||
| func New(p client.ConfigProvider, bucket string, keyPrefix string) *Backend { | ||
| return &Backend{ | ||
| s3: s3.New(p), | ||
| uploader: s3manager.NewUploader(p), | ||
| downloader: s3manager.NewDownloader(p), | ||
| bucket: bucket, | ||
| keyPrefix: keyPrefix, | ||
| } | ||
| } | ||
|
|
||
| func (b *Backend) CopyPlans(repoDir string, repoFullName string, env string, pullNum int) ([]plan.Plan, error) { | ||
| // first list the plans with the correct prefix | ||
| prefix := pathutil.Join(b.keyPrefix, repoFullName, strconv.Itoa(pullNum)) | ||
| list, err := b.s3.ListObjects(&s3.ListObjectsInput{Bucket: aws.String(b.bucket), Prefix: &prefix}) | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "listing plans") | ||
| } | ||
|
|
||
| var plans []plan.Plan | ||
| for _, obj := range list.Contents { | ||
| planName := pathutil.Base(*obj.Key) | ||
|
|
||
| // only get plans from the correct env | ||
| if planName != env+".tfplan" { | ||
| continue | ||
| } | ||
|
|
||
| // determine the path relative to the repo | ||
| relPath, err := filepath.Rel(prefix, *obj.Key) | ||
| if err != nil { | ||
| continue | ||
| } | ||
| downloadPath := filepath.Join(repoDir, relPath) | ||
| file, err := os.Create(downloadPath) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "creating file %s to download plan to", downloadPath) | ||
| } | ||
| defer file.Close() | ||
|
|
||
| _, err = b.downloader.Download(file, | ||
| &s3.GetObjectInput{ | ||
| Bucket: aws.String(b.bucket), | ||
| Key: obj.Key, | ||
| }) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "downloading file at %s", *obj.Key) | ||
| } | ||
| plans = append(plans, plan.Plan{ | ||
| Project: models.Project{ | ||
| Path: pathutil.Dir(relPath), | ||
| RepoFullName: repoFullName, | ||
| }, | ||
| LocalPath: downloadPath, | ||
| }) | ||
| } | ||
| return plans, nil | ||
| } | ||
|
|
||
| func (b *Backend) SavePlan(path string, project models.Project, env string, pullNum int) error { | ||
| f, err := os.Open(path) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "opening plan at %s", path) | ||
| } | ||
|
|
||
| key := pathutil.Join(b.keyPrefix, project.RepoFullName, strconv.Itoa(pullNum), project.Path, env+".tfplan") | ||
| _, err = b.uploader.Upload(&s3manager.UploadInput{ | ||
| Bucket: aws.String(b.bucket), | ||
| Key: &key, | ||
| Body: f, | ||
| Metadata: map[string]*string{ | ||
| "repoFullName": aws.String(project.RepoFullName), | ||
| "path": aws.String(project.Path), | ||
| "env": aws.String(env), | ||
| "pullNum": aws.String(strconv.Itoa(pullNum)), | ||
| }, | ||
| }) | ||
| if err != nil { | ||
| return errors.Wrap(err, "uploading plan to s3") | ||
| } | ||
| return nil | ||
| } |
Oops, something went wrong.