diff --git a/AUTHORS b/AUTHORS index 2a42fb9a4..66c04b1a6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -64,3 +64,4 @@ List of contributors, in chronological order: * Golf Hu (https://github.com/hudeng-go) * Cookie Fei (https://github.com/wuhuang26) * Andrey Loukhnov (https://github.com/aol-nnov) +* Blake Kostner (https://github.com/btkostner) diff --git a/api/publish.go b/api/publish.go index c7aced61d..865e14ee4 100644 --- a/api/publish.go +++ b/api/publish.go @@ -240,7 +240,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err) } @@ -337,6 +337,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { resources = append(resources, string(published.Key())) taskName := fmt.Sprintf("Update published %s (%s): %s", published.SourceKind, strings.Join(updatedComponents, " "), strings.Join(updatedSnapshots, ", ")) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { +<<<<<<< HEAD err = collection.LoadComplete(published, collectionFactory) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err) @@ -362,7 +363,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { } } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } diff --git a/cmd/publish_snapshot.go b/cmd/publish_snapshot.go index 3d2e43e60..f6e210468 100644 --- a/cmd/publish_snapshot.go +++ b/cmd/publish_snapshot.go @@ -166,7 +166,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n") } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath()) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index 0f1a620b9..f9fe72d3e 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -99,7 +99,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error { published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath()) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/cmd/publish_update.go b/cmd/publish_update.go index 28de8c67c..8ebcc0075 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -68,7 +68,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error { published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite) + err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath()) if err != nil { return fmt.Errorf("unable to publish: %s", err) } diff --git a/context/context.go b/context/context.go index 30e84d42f..df50bc60b 100644 --- a/context/context.go +++ b/context/context.go @@ -529,6 +529,11 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier { return pgp.NewGpgVerifier(context.getGPGFinder()) } +// SkelPath builds the local skeleton folder +func (context *AptlyContext) SkelPath() string { + return filepath.Join(context.config().RootDir, "skel") +} + // UpdateFlags sets internal copy of flags in the context func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) { context.Lock() diff --git a/deb/index_files.go b/deb/index_files.go index 01de3dd53..27a194703 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -389,6 +389,27 @@ func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile return file } +func (files *indexFiles) SkelIndex(component, path string) *indexFile { + key := fmt.Sprintf("si-%s-%s", component, path) + file, ok := files.indexes[key] + + if !ok { + relativePath := filepath.Join(component, path) + + file = &indexFile{ + parent: files, + discardable: false, + compressable: false, + onlyGzip: false, + relativePath: relativePath, + } + + files.indexes[key] = file + } + + return file +} + func (files *indexFiles) ReleaseFile() *indexFile { return &indexFile{ parent: files, diff --git a/deb/publish.go b/deb/publish.go index 79c47cb83..b61d641e8 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -550,9 +550,47 @@ func (p *PublishedRepo) GetCodename() string { return p.Codename } +// GetSkelFiles returns a map of files to be added to a repo. Key being the relative +// path from component folder, and value being the full local FS path. +func (p *PublishedRepo) GetSkelFiles(skelDir string, component string) (map[string]string, error) { + files := make(map[string]string) + + if skelDir == "" { + return files, nil + } + + fsPath := filepath.Join(skelDir, p.Prefix, "dists", p.Distribution, component) + if err := filepath.Walk(fsPath, func(path string, _ os.FileInfo, err error) error { + if err != nil { + return err + } + + stat, err := os.Stat(path) + if err != nil { + return err + } + + if !stat.Mode().IsRegular() { + return nil + } + + relativePath, err := filepath.Rel(fsPath, path) + if err != nil { + return err + } + + files[relativePath] = path + return nil + }); err != nil && !os.IsNotExist(err) { + return files, err + } + + return files, nil +} + // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, - collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error { + collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite, skelDir string) error { publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool")) @@ -761,6 +799,30 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } } + for component := range p.sourceItems { + skelFiles, err := p.GetSkelFiles(skelDir, component) + if err != nil { + return fmt.Errorf("unable to get skeleton files: %v", err) + } + + for relPath, absPath := range skelFiles { + bufWriter, err := indexes.SkelIndex(component, relPath).BufWriter() + if err != nil { + return fmt.Errorf("unable to generate skeleton index: %v", err) + } + + file, err := os.Open(absPath) + if err != nil { + return fmt.Errorf("unable to read skeleton file: %v", err) + } + + _, err = bufio.NewReader(file).WriteTo(bufWriter) + if err != nil { + return fmt.Errorf("unable to write skeleton file: %v", err) + } + } + } + udebs := []bool{false} if hadUdebs { udebs = append(udebs, true) diff --git a/deb/publish_test.go b/deb/publish_test.go index 5243a41a6..f618fc57d 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -360,7 +360,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) { } func (s *PublishedRepoSuite) TestPublish(c *C) { - err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false) + err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "") c.Assert(err, IsNil) c.Check(s.repo.Architectures, DeepEquals, []string{"i386"}) @@ -407,7 +407,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) { } func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { - err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists) @@ -415,7 +415,7 @@ func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { } func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) { - err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists) @@ -423,7 +423,7 @@ func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) { } func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) { - err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo4.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"), PathExists) @@ -431,7 +431,7 @@ func (s *PublishedRepoSuite) TestPublishLocalSourceRepo(c *C) { } func (s *PublishedRepoSuite) TestPublishOtherStorage(c *C) { - err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false) + err := s.repo5.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") c.Assert(err, IsNil) c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/maverick/Release"), PathExists) diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index baab5ff6b..0b7937c39 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -121,7 +121,8 @@ Options: * `rootDir`: is root of directory storage to store database (`rootDir`/db), the default for downloaded packages (`rootDir`/pool) and - the default for published repositories (`rootDir`/public) + the default for published repositories (`rootDir`/public) and + skeleton files (`rootDir`/skel) * `databaseBackend`: the database config; if this config is empty, use levledb backend by default diff --git a/system/lib.py b/system/lib.py index e162b964e..17f608734 100644 --- a/system/lib.py +++ b/system/lib.py @@ -404,6 +404,15 @@ def check_cmd_output(self, command, gold_name, match_prepare=None, expected_code else: raise + def write_file(self, path, content): + full_path = os.path.join(os.environ["HOME"], ".aptly", path) + + if not os.path.exists(os.path.dirname(full_path)): + os.makedirs(os.path.dirname(full_path), 0o755) + + with open(full_path, "w") as f: + f.write(content) + def read_file(self, path, mode=''): with open(os.path.join(os.environ["HOME"], self.aptlyDir, path), "r" + mode) as f: return f.read() diff --git a/system/t06_publish/PublishRepo34Test_gold b/system/t06_publish/PublishRepo34Test_gold new file mode 100644 index 000000000..365295fa9 --- /dev/null +++ b/system/t06_publish/PublishRepo34Test_gold @@ -0,0 +1,14 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: + +Local repo local-repo has been successfully published. +Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing. +Now you can add following line to apt sources: + deb http://your-server/ maverick main + deb-src http://your-server/ maverick main +Don't forget to add your GPG key to apt with apt-key. + +You can also use `aptly serve` to publish your repositories over HTTP quickly. diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index 7e4b10c51..20fbdf1bf 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -888,3 +888,61 @@ def check(self): self.check_exists('public/dists/maverick/main/binary-amd64/Packages') self.check_exists('public/dists/maverick/main/binary-amd64/Packages.gz') self.check_not_exists('public/dists/maverick/main/binary-amd64/Packages.bz2') + + +class PublishRepo34Test(BaseTest): + """ + publish repo: skeleton files + """ + fixtureCmds = [ + "aptly repo create local-repo", + "aptly repo add local-repo ${files}" + ] + runCmd = "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -skip-contents local-repo" + gold_processor = BaseTest.expand_environ + + def prepare_fixture(self): + super(PublishRepo34Test, self).prepare_fixture() + + self.write_file(os.path.join('skel', 'dists', 'maverick', 'main', 'dep11', 'README'), 'README test file') + + def check(self): + super(PublishRepo34Test, self).check() + + self.check_exists('public/dists/maverick/main/dep11/README') + + self.check_exists('public/dists/maverick/Release') + + readme = self.read_file('public/dists/maverick/main/dep11/README') + if readme != 'README test file': + raise Exception("README file not copied on publish") + + release = self.read_file('public/dists/maverick/Release').split("\n") + release = [l for l in release if l.startswith(" ")] + pathsSeen = set() + for l in release: + fileHash, fileSize, path = l.split() + pathsSeen.add(path) + + fileSize = int(fileSize) + + st = os.stat(os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/', path)) + if fileSize != st.st_size: + raise Exception("file size doesn't match for %s: %d != %d" % (path, fileSize, st.st_size)) + + if len(fileHash) == 32: + h = hashlib.md5() + elif len(fileHash) == 40: + h = hashlib.sha1() + elif len(fileHash) == 64: + h = hashlib.sha256() + else: + h = hashlib.sha512() + + h.update(self.read_file(os.path.join('public/dists/maverick', path), mode='b')) + + if h.hexdigest() != fileHash: + raise Exception("file hash doesn't match for %s: %s != %s" % (path, fileHash, h.hexdigest())) + + if 'main/dep11/README' not in pathsSeen: + raise Exception("README file not included in release file")