diff --git a/bzz/api.go b/bzz/api.go index 7bdfe8f5d622..1dc1766689aa 100644 --- a/bzz/api.go +++ b/bzz/api.go @@ -226,6 +226,8 @@ func (self *Api) Modify(rootHash, path, contentHash, contentType string) (newRoo return fmt.Sprintf("%064x", trie.hash), nil } +const maxParallelFiles = 5 + // Download replicates the manifest path structure on the local filesystem // under localpath func (self *Api) Download(bzzpath, localpath string) (err error) { @@ -264,30 +266,80 @@ func (self *Api) Download(bzzpath, localpath string) (err error) { return } + type downloadListEntry struct { + key Key + path string + } + + var list []*downloadListEntry + var mde, mderr error + prevPath := lpath - trie.listWithPrefix(path, func(entry *manifestTrieEntry, suffix string) { // TODO: paralellize + err = trie.listWithPrefix(path, func(entry *manifestTrieEntry, suffix string) { // TODO: paralellize key := common.Hex2Bytes(entry.Hash) - reader := self.dpa.Retrieve(key) path := lpath + "/" + suffix dir := filepath.Dir(path) if dir != prevPath { - os.MkdirAll(dir, os.ModePerm) // TODO: handle errors + mde = os.MkdirAll(dir, os.ModePerm) + if mde != nil { + mderr = mde + } prevPath = dir } - f, _ := os.Create(path) // TODO: handle errors, ??path separators - writer := bufio.NewWriter(f) - //io.Copy(writer, reader) // TODO: handle errors - io.CopyN(writer, reader, reader.Size()) // TODO: handle errors - - writer.Flush() - f.Close() + if (mde == nil) && (path != dir+"/") { + list = append(list, &downloadListEntry{key: key, path: path}) + } }) + if err == nil { + err = mderr + } + + cnt := len(list) + errors := make([]error, cnt) + done := make(chan bool, maxParallelFiles) + dcnt := 0 + for i, entry := range list { + if i >= dcnt+maxParallelFiles { + <-done + dcnt++ + } + go func(i int, entry *downloadListEntry, done chan bool) { + f, err := os.Create(entry.path) // TODO: path separators + if err == nil { + reader := self.dpa.Retrieve(entry.key) + writer := bufio.NewWriter(f) + _, err = io.CopyN(writer, reader, reader.Size()) // TODO: handle errors + err2 := writer.Flush() + if err == nil { + err = err2 + } + err2 = f.Close() + if err == nil { + err = err2 + } + } + + errors[i] = err + done <- true + }(i, entry, done) + } + for dcnt < cnt { + <-done + dcnt++ + } + + if err != nil { + return + } + for i, _ := range list { + if errors[i] != nil { + return errors[i] + } + } return } -const maxParallelFiles = 5 - // Upload replicates a local directory as a manifest file and uploads it // using dpa store // TODO: localpath should point to a manifest diff --git a/bzz/api_test.go b/bzz/api_test.go index b69093ffb7c3..87b2ac94100c 100644 --- a/bzz/api_test.go +++ b/bzz/api_test.go @@ -94,6 +94,47 @@ func TestApiDirUpload(t *testing.T) { } } +func TestApiDirUploadModify(t *testing.T) { + api, err := testApi() + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + bzzhash, err := api.Upload(path.Join(testDir, "test0"), "") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + bzzhash, err = api.Modify(bzzhash, "index.html", "", "") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + bzzhash, err = api.Modify(bzzhash, "index2.html", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + bzzhash, err = api.Modify(bzzhash, "img/logo.png", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + content, err := ioutil.ReadFile(path.Join(testDir, "test0", "index.html")) + testGet(t, api, path.Join(bzzhash, "index2.html"), content, "text/html; charset=utf-8", 0, 202) + testGet(t, api, path.Join(bzzhash, "img", "logo.png"), content, "text/html; charset=utf-8", 0, 202) + + content, err = ioutil.ReadFile(path.Join(testDir, "test0", "index.css")) + testGet(t, api, path.Join(bzzhash, "index.css"), content, "text/css", 0, 132) + + _, _, _, _, err = api.Get(bzzhash) + if err == nil { + t.Errorf("expected error: %v", err) + } +} + func TestApiDirUploadWithRootFile(t *testing.T) { api, err := testApi() if err != nil { diff --git a/bzz/js_api.go b/bzz/js_api.go index ae5ca8d2f359..fa6ff7d1cd70 100644 --- a/bzz/js_api.go +++ b/bzz/js_api.go @@ -23,6 +23,7 @@ func NewJSApi(vm *jsre.JSRE, api *Api) (jsapi *JSApi) { o.Set("upload", jsapi.upload) o.Set("get", jsapi.get) o.Set("put", jsapi.put) + o.Set("modify", jsapi.modify) return } @@ -211,5 +212,42 @@ func (self *JSApi) upload(call otto.FunctionCall) otto.Value { return v } -// http.PostForm("http://example.com/form", -// url.Values{"key": {"Value"}, "id": {"123"}}) +func (self *JSApi) modify(call otto.FunctionCall) otto.Value { + argc := len(call.ArgumentList) + if (argc != 2) && (argc != 4) { + fmt.Println("requires 2 or 4 arguments: bzz.modify(rootHash, path[, contentHash, contentType])") + return otto.UndefinedValue() + } + root, err := call.Argument(0).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + path, err := call.Argument(1).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + var chash, ctype string + if argc == 4 { + chash, err = call.Argument(2).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + ctype, err = call.Argument(3).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + } + + res, err := self.api.Modify(root, path, chash, ctype) + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + + v, _ := call.Otto.ToValue(res) + return v +} diff --git a/bzz/manifest.go b/bzz/manifest.go index d67f7c33da84..6f91c722e7ec 100644 --- a/bzz/manifest.go +++ b/bzz/manifest.go @@ -211,7 +211,7 @@ func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry) (err error) { return } -func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *manifestTrieEntry, suffix string)) { +func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *manifestTrieEntry, suffix string)) (err error) { plen := len(prefix) var start, stop int if plen == 0 { @@ -231,8 +231,13 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma if epl < l { l = epl } - if (prefix[:l] == entry.Path[:l]) && (self.loadSubTrie(entry) == nil) { // TODO: handle errors - entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], cb) + if prefix[:l] == entry.Path[:l] { + sterr := self.loadSubTrie(entry) + if sterr == nil { + entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], cb) + } else { + err = sterr + } } } else { if (epl >= plen) && (prefix == entry.Path[:plen]) { @@ -241,10 +246,11 @@ func (self *manifestTrie) listWithPrefixInt(prefix, rp string, cb func(entry *ma } } } + return } -func (self *manifestTrie) listWithPrefix(prefix string, cb func(entry *manifestTrieEntry, suffix string)) { - self.listWithPrefixInt(prefix, "", cb) +func (self *manifestTrie) listWithPrefix(prefix string, cb func(entry *manifestTrieEntry, suffix string)) (err error) { + return self.listWithPrefixInt(prefix, "", cb) } func (self *manifestTrie) findPrefixOf(path string) (entry *manifestTrieEntry, pos int) {