diff --git a/.circleci/config.yml b/.circleci/config.yml index 8891a342da8..8c7cc07e949 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -229,7 +229,7 @@ jobs: - run: name: Cloning command: | - git clone https://github.com/ipfs/go-ipfs-api.git + git clone -b ipld-in-ipfs https://github.com/ipfs/go-ipfs-api.git git -C go-ipfs-api log -1 - run: name: Starting the daemon diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 005847039d7..42a18336b5b 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -77,10 +77,10 @@ into an object of the specified format. cmds.FileArg("object data", true, true, "The object to put").EnableStdin(), }, Options: []cmds.Option{ - cmds.StringOption("format", "f", "Format that the object will be added as.").WithDefault("cbor"), - cmds.StringOption("input-enc", "Format that the input object will be.").WithDefault("json"), + cmds.StringOption("format", "f", "Format that the object will be added as.").WithDefault("dag-cbor"), + cmds.StringOption("input-enc", "Format that the input object will be.").WithDefault("dag-json"), cmds.BoolOption("pin", "Pin this object when adding."), - cmds.StringOption("hash", "Hash function to use").WithDefault(""), + cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"), }, Run: dagPut, Type: OutputObject{}, @@ -108,6 +108,9 @@ format. Arguments: []cmds.Argument{ cmds.StringArg("ref", true, false, "The object to get").EnableStdin(), }, + Options: []cmds.Option{ + cmds.StringOption("format", "f", "Format that the object will be serialized as.").WithDefault("dag-json"), + }, Run: dagGet, } diff --git a/core/commands/dag/get.go b/core/commands/dag/get.go index a5f92273ae3..c5c983e09b5 100644 --- a/core/commands/dag/get.go +++ b/core/commands/dag/get.go @@ -1,11 +1,18 @@ package dagcmd import ( - "strings" + "fmt" + "io" "github.com/ipfs/go-ipfs/core/commands/cmdenv" + ipldlegacy "github.com/ipfs/go-ipld-legacy" "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/multicodec" + "github.com/ipld/go-ipld-prime/traversal" + mc "github.com/multiformats/go-multicodec" + cmds "github.com/ipfs/go-ipfs-cmds" ) @@ -15,6 +22,12 @@ func dagGet(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e return err } + format, _ := req.Options["format"].(string) + var fCodec mc.Code + if err := fCodec.Set(format); err != nil { + return err + } + rp, err := api.ResolvePath(req.Context, path.New(req.Arguments[0])) if err != nil { return err @@ -25,14 +38,34 @@ func dagGet(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e return err } - var out interface{} = obj + universal, ok := obj.(ipldlegacy.UniversalNode) + if !ok { + return fmt.Errorf("%T is not a valid IPLD node", obj) + } + + finalNode := universal.(ipld.Node) + if len(rp.Remainder()) > 0 { - rem := strings.Split(rp.Remainder(), "/") - final, _, err := obj.Resolve(rem) + remainderPath := ipld.ParsePath(rp.Remainder()) + + finalNode, err = traversal.Get(finalNode, remainderPath) if err != nil { return err } - out = final } - return cmds.EmitOnce(res, &out) + + encoder, err := multicodec.LookupEncoder(uint64(fCodec)) + if err != nil { + return fmt.Errorf("invalid encoding: %s - %s", format, err) + } + + r, w := io.Pipe() + go func() { + defer w.Close() + if err := encoder(finalNode, w); err != nil { + _ = res.CloseWithError(err) + } + }() + + return res.Emit(r) } diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index 7f6e744c872..9319f7f8a18 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -1,16 +1,28 @@ package dagcmd import ( + "bytes" "fmt" - "math" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs/core/commands/cmdenv" - "github.com/ipfs/go-ipfs/core/coredag" + ipldlegacy "github.com/ipfs/go-ipld-legacy" + "github.com/ipld/go-ipld-prime/multicodec" + basicnode "github.com/ipld/go-ipld-prime/node/basic" cmds "github.com/ipfs/go-ipfs-cmds" files "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" - mh "github.com/multiformats/go-multihash" + mc "github.com/multiformats/go-multicodec" + + // Expected minimal set of available format/ienc codecs. + _ "github.com/ipld/go-codec-dagpb" + _ "github.com/ipld/go-ipld-prime/codec/cbor" + _ "github.com/ipld/go-ipld-prime/codec/dagcbor" + _ "github.com/ipld/go-ipld-prime/codec/dagjson" + _ "github.com/ipld/go-ipld-prime/codec/json" + _ "github.com/ipld/go-ipld-prime/codec/raw" ) func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { @@ -24,16 +36,33 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e hash, _ := req.Options["hash"].(string) dopin, _ := req.Options["pin"].(bool) - // mhType tells inputParser which hash should be used. MaxUint64 means 'use - // default hash' (sha256 for cbor, sha1 for git..) - mhType := uint64(math.MaxUint64) + var icodec mc.Code + if err := icodec.Set(ienc); err != nil { + return err + } + var fcodec mc.Code + if err := fcodec.Set(format); err != nil { + return err + } + var mhType mc.Code + if err := mhType.Set(hash); err != nil { + return err + } + + cidPrefix := cid.Prefix{ + Version: 1, + Codec: uint64(fcodec), + MhType: uint64(mhType), + MhLength: -1, + } - if hash != "" { - var ok bool - mhType, ok = mh.Names[hash] - if !ok { - return fmt.Errorf("%s in not a valid multihash name", hash) - } + decoder, err := multicodec.LookupDecoder(uint64(icodec)) + if err != nil { + return err + } + encoder, err := multicodec.LookupEncoder(uint64(fcodec)) + if err != nil { + return err } var adder ipld.NodeAdder = api.Dag() @@ -48,22 +77,44 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e if file == nil { return fmt.Errorf("expected a regular file") } - nds, err := coredag.ParseInputs(ienc, format, file, mhType, -1) + + node := basicnode.Prototype.Any.NewBuilder() + if err := decoder(node, file); err != nil { + return err + } + n := node.Build() + + bd := bytes.NewBuffer([]byte{}) + if err := encoder(n, bd); err != nil { + return err + } + + blockCid, err := cidPrefix.Sum(bd.Bytes()) + if err != nil { + return err + } + blk, err := blocks.NewBlockWithCid(bd.Bytes(), blockCid) if err != nil { return err } - if len(nds) == 0 { - return fmt.Errorf("no node returned from ParseInputs") + ln := ipldlegacy.LegacyNode{ + Block: blk, + Node: n, } - for _, nd := range nds { - err := b.Add(req.Context, nd) - if err != nil { - return err - } + if err := b.Add(req.Context, &ln); err != nil { + return err } + /* + for _, nd := range nds { + err := b.Add(req.Context, nd) + if err != nil { + return err + } + } + */ - cid := nds[0].Cid() + cid := ln.Cid() if err := res.Emit(&OutputObject{Cid: cid}); err != nil { return err } diff --git a/go.mod b/go.mod index d67cd91587e..a123bfe0de3 100644 --- a/go.mod +++ b/go.mod @@ -46,22 +46,23 @@ require ( github.com/ipfs/go-ipfs-util v0.0.2 github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 - github.com/ipfs/go-ipld-git v0.0.3 + github.com/ipfs/go-ipld-git v0.0.4-0.20210726071918-f3dcf6c6f96e + github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 github.com/ipfs/go-ipns v0.0.2 github.com/ipfs/go-log v1.0.4 github.com/ipfs/go-merkledag v0.3.3-0.20210325015807-e952d22343f9 github.com/ipfs/go-metrics-interface v0.0.1 github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 - github.com/ipfs/go-path v0.0.10-0.20210421213242-802a897dfcd8 + github.com/ipfs/go-path v0.0.10-0.20210726072238-b450033b62d9 github.com/ipfs/go-pinning-service-http-client v0.1.0 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-unixfsnode v1.1.1 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62 - github.com/ipld/go-car v0.2.1-0.20210312021557-7afab98d034f + github.com/ipld/go-car v0.3.1 github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 - github.com/ipld/go-ipld-prime v0.9.1-0.20210402181957-7406578571d1 + github.com/ipld/go-ipld-prime v0.10.1-0.20210727022528-083c39518830 github.com/jbenet/go-is-domain v1.0.5 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 @@ -96,6 +97,7 @@ require ( github.com/multiformats/go-multiaddr v0.3.1 github.com/multiformats/go-multiaddr-dns v0.2.0 github.com/multiformats/go-multibase v0.0.3 + github.com/multiformats/go-multicodec v0.2.1-0.20210723011827-52a3ac546597 github.com/multiformats/go-multihash v0.0.15 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index f8ca1e9d161..959a03ec19f 100644 --- a/go.sum +++ b/go.sum @@ -449,8 +449,8 @@ github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dC github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= -github.com/ipfs/go-ipld-git v0.0.3 h1:/YjkjCyo5KYRpW+suby8Xh9Cm/iH9dAgGV6qyZ1dGus= -github.com/ipfs/go-ipld-git v0.0.3/go.mod h1:RuvMXa9qtJpDbqngyICCU/d+cmLFXxLsbIclmD0Lcr0= +github.com/ipfs/go-ipld-git v0.0.4-0.20210726071918-f3dcf6c6f96e h1:nIn/bK6M+x9YjgDIs0JU9BexgFlOiSXA5MnVIOK3yhM= +github.com/ipfs/go-ipld-git v0.0.4-0.20210726071918-f3dcf6c6f96e/go.mod h1:DHEYwGO2tc27/7VVRtNqFvJBFbLufPW75eGkvYJgNcU= github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 h1:PXjRvJkxqQ2EbwviF5GEURYqvhS/ulee/ZdVsVgVtHM= github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipns v0.0.2 h1:oq4ErrV4hNQ2Eim257RTYRgfOSV/s8BDaf9iIl4NwFs= @@ -485,8 +485,8 @@ github.com/ipfs/go-mfs v0.1.2 h1:DlelNSmH+yz/Riy0RjPKlooPg0KML4lXGdLw7uZkfAg= github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4diFo0= github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno= github.com/ipfs/go-path v0.0.10-0.20210405201800-40f1060226f7/go.mod h1:g8egwymo/MjfPUu/ojghk4YQKwjn6MGO+UUv4eUw08M= -github.com/ipfs/go-path v0.0.10-0.20210421213242-802a897dfcd8 h1:YT89FH52ynHRCZJIM95z+/4fz0rQ7nFPFLRufOzm9AE= -github.com/ipfs/go-path v0.0.10-0.20210421213242-802a897dfcd8/go.mod h1:YMCaFyukoM1bSaPm1eL9Q6IPr0z4WytNeVKlBSm8a0c= +github.com/ipfs/go-path v0.0.10-0.20210726072238-b450033b62d9 h1:GEptmVxRDOWObHiqZPbk9PyLlAwyZyCrBc4hu3oGqeI= +github.com/ipfs/go-path v0.0.10-0.20210726072238-b450033b62d9/go.mod h1:grgssIrSAXeXl3OQ7PfI3yMPbD6v84HrvLjkkFuvIdc= github.com/ipfs/go-peertaskqueue v0.0.4/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= @@ -505,21 +505,18 @@ github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZ github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o= github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62 h1:PJ5gH7jjJtQ8OT+iACcolpMnvhCpfRtpegWfMF785uk= github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62/go.mod h1:leFsW1rX9F7+0N9Tss2kR82w9oafDmczBdE8z4VWGRo= -github.com/ipld/go-car v0.2.1-0.20210312021557-7afab98d034f h1:417+6v+keLsNoVi2vPJamDsW/ewuAs2uE3hlFt5muGs= -github.com/ipld/go-car v0.2.1-0.20210312021557-7afab98d034f/go.mod h1:kHunAcD305JwLqwI9MfKvuQu4ljwWRZQWfDWPrkkhcg= -github.com/ipld/go-codec-dagpb v1.0.2-0.20210308154810-d05d02fa186e/go.mod h1:oYexiw3WkBIVD5UTNkVuOd0iyEcLxqytAQa90F3nH9M= +github.com/ipld/go-car v0.3.1 h1:WT+3cdmXlvmWOlGxk9webhj4auGO5QvgqC2vCCkFRXs= +github.com/ipld/go-car v0.3.1/go.mod h1:dPkEWeAK8KaVvH5TahaCs6Mncpd4lDMpkbs0/SPzuVs= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.2.1-0.20210330082435-8ec6b0fbad18/go.mod h1:GMLfso6KSkYJlIbd2cGKdGMe/hM5/IukeXRQ+u6zTrQ= github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 h1:m/ZZEoOdswHrrcikTC+fX4x6tnevJs0hoyNzijlT41A= github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2/go.mod h1:GMLfso6KSkYJlIbd2cGKdGMe/hM5/IukeXRQ+u6zTrQ= -github.com/ipld/go-ipld-prime v0.7.1-0.20210225173718-8fef5312eb12/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= -github.com/ipld/go-ipld-prime v0.7.1-0.20210312004928-8a500e6b8a62/go.mod h1:ZwznT3awHhuBoB0SgGxSo8SZEg6XbQtsZ6WahL9r9z0= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.9.1-0.20210402181957-7406578571d1 h1:dIKSj9r+CCXz9t6p4TfbDx34CfSjLfasRZf37SXlNjg= github.com/ipld/go-ipld-prime v0.9.1-0.20210402181957-7406578571d1/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.10.1-0.20210723093545-ebe39c7bab39/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.10.1-0.20210727022528-083c39518830 h1:tQyWv1pyxNIFe2hPczHGITClSQRJlW6dLIcEvkdSRgY= +github.com/ipld/go-ipld-prime v0.10.1-0.20210727022528-083c39518830/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -961,6 +958,8 @@ github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multicodec v0.2.1-0.20210723011827-52a3ac546597 h1:ULcqjrvbMRf7TiRiXlmXprupBxpeUkL8q2N8Xq+bNi4= +github.com/multiformats/go-multicodec v0.2.1-0.20210723011827-52a3ac546597/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1269,12 +1268,10 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1409,7 +1406,6 @@ golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1421,7 +1417,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= diff --git a/plugin/ipld.go b/plugin/ipld.go index 5b45e9cd333..38d1484208a 100644 --- a/plugin/ipld.go +++ b/plugin/ipld.go @@ -8,6 +8,9 @@ import ( // PluginIPLD is an interface that can be implemented to add handlers for // for different IPLD formats +// Deprecated: Codecs can now be registered directly in a Plugin's Init method +// using github.com/ipld/go-ipld-prime/multicodec.RegisterEncoder and +// github.com/ipld/go-ipld-prime/multicodec.RegisterDecoder. type PluginIPLD interface { Plugin diff --git a/plugin/plugins/git/git.go b/plugin/plugins/git/git.go index 23b79ad59a1..e1d38e5b7d0 100644 --- a/plugin/plugins/git/git.go +++ b/plugin/plugins/git/git.go @@ -2,17 +2,15 @@ package git import ( "compress/zlib" - "fmt" "io" - "math" - "github.com/ipfs/go-ipfs/core/coredag" "github.com/ipfs/go-ipfs/plugin" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + // Note that depending on this package registers it's multicodec encoder and decoder. git "github.com/ipfs/go-ipld-git" - mh "github.com/multiformats/go-multihash" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/multicodec" + mc "github.com/multiformats/go-multicodec" ) // Plugins is exported list of plugins that will be loaded @@ -22,7 +20,7 @@ var Plugins = []plugin.Plugin{ type gitPlugin struct{} -var _ plugin.PluginIPLD = (*gitPlugin)(nil) +var _ plugin.Plugin = (*gitPlugin)(nil) func (*gitPlugin) Name() string { return "ipld-git" @@ -33,43 +31,19 @@ func (*gitPlugin) Version() string { } func (*gitPlugin) Init(_ *plugin.Environment) error { + // register a custom identifier in the reserved range for import of "zlib-encoded git objects." + // TODO: give this a name. + multicodec.RegisterDecoder(uint64(mc.ReservedStart+mc.GitRaw), decodeZlibGit) return nil } -func (*gitPlugin) RegisterBlockDecoders(dec format.BlockDecoder) error { - dec.Register(cid.GitRaw, git.DecodeBlock) - return nil -} - -func (*gitPlugin) RegisterInputEncParsers(iec coredag.InputEncParsers) error { - iec.AddParser("raw", "git", parseRawGit) - iec.AddParser("zlib", "git", parseZlibGit) - return nil -} - -func parseRawGit(r io.Reader, mhType uint64, mhLen int) ([]format.Node, error) { - if mhType != math.MaxUint64 && mhType != mh.SHA1 { - return nil, fmt.Errorf("unsupported mhType %d", mhType) - } - - if mhLen != -1 && mhLen != mh.DefaultLengths[mh.SHA1] { - return nil, fmt.Errorf("invalid mhLen %d", mhLen) - } - - nd, err := git.ParseObject(r) - if err != nil { - return nil, err - } - - return []format.Node{nd}, nil -} - -func parseZlibGit(r io.Reader, mhType uint64, mhLen int) ([]format.Node, error) { +func decodeZlibGit(na ipld.NodeAssembler, r io.Reader) error { rc, err := zlib.NewReader(r) if err != nil { - return nil, err + return err } defer rc.Close() - return parseRawGit(rc, mhType, mhLen) + + return git.Decode(na, rc) } diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 2225f79ec4d..b95e853cd8b 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -25,31 +25,38 @@ test_expect_success "make an ipld object in json" ' printf "{\"hello\":\"world\",\"cats\":[{\"/\":\"%s\"},{\"water\":{\"/\":\"%s\"}}],\"magic\":{\"/\":\"%s\"},\"sub\":{\"dict\":\"ionary\",\"beep\":[0,\"bop\"]}}" $HASH1 $HASH2 $HASH3 > ipld_object ' +test_expect_success "make the same ipld object in dag-cbor, dag-json and dag-pb" ' + echo "omREYXRhT6JkRGF0YWF4ZUxpbmtzgGVMaW5rc4A=" | base64 -d > ipld_object_dagcbor + echo "Cg+iZERhdGFheGVMaW5rc4A=" | base64 -d > ipld_object_dagpb + echo "{\"Data\":{\"/\":{\"bytes\":\"omREYXRhYXhlTGlua3OA\"}},\"Links\":[]}" > ipld_object_dagjson +' + test_dag_cmd() { - test_expect_success "can add an ipld object using protobuf" ' - IPLDHASH=$(cat ipld_object | ipfs dag put -f protobuf) + test_expect_success "can add an ipld object using dag-json to dag-cbor" ' + IPLDHASH=$(cat ipld_object | ipfs dag put --input-enc=dag-json -f dag-cbor) ' - test_expect_success "output looks correct" ' - EXPHASH="QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" + test_expect_success "CID looks correct" ' + EXPHASH="bafyreiblwimnjbqcdoeafiobk6q27jcw64ew7n2fmmhdpldd63edmjecde" + echo $IPLDHASH > ipld_hash && \ test $EXPHASH = $IPLDHASH ' - test_expect_success "can add an ipld object using protobuf and --cid=base=base32" ' - IPLDHASHb32=$(cat ipld_object | ipfs dag put -f protobuf --cid-base=base32) + test_expect_success "can add an ipld object using defaults" ' + IPLDHASH=$(cat ipld_object | ipfs dag put) ' - test_expect_success "output looks correct (does not upgrade to CIDv1)" ' - test $EXPHASH = $IPLDHASHb32 + test_expect_success "CID looks correct" ' + EXPHASH="bafyreiblwimnjbqcdoeafiobk6q27jcw64ew7n2fmmhdpldd63edmjecde" + test $EXPHASH = $IPLDHASH ' - test_expect_success "can add an ipld object" ' - IPLDHASH=$(cat ipld_object | ipfs dag put) + test_expect_success "can add an ipld object using protobuf and --cid=base=base32" ' + IPLDHASHb32=$(cat ipld_object | ipfs dag put --input-enc=dag-json -f dag-cbor --cid-base=base32) ' - test_expect_success "output looks correct" ' - EXPHASH="bafyreidjtjfmavdk7epvztob2m5vlm3pxp3gjmpyewro4qlbw5n4f4iz64" - test $EXPHASH = $IPLDHASH + test_expect_success "output looks correct (does not upgrade to CIDv1)" ' + test $EXPHASH = $IPLDHASHb32 ' test_expect_success "can add an ipld object using --cid-base=base32" ' @@ -60,6 +67,33 @@ test_dag_cmd() { test $(ipfs cid base32 $EXPHASH) = $IPLDHASHb32 ' + test_expect_success "can add an ipld object using dag-cbor to dag-pb" ' + IPLDPBHASH=$(cat ipld_object_dagcbor | ipfs dag put --input-enc=dag-cbor -f dag-pb) + ' + + test_expect_success "dag-pb CID looks correct" ' + EXPHASH="bafybeicbvnvha5b25w77plipz3v5axqydbhxb2s2ojglexotowwieoecvy" + test $EXPHASH = $IPLDPBHASH + ' + + test_expect_success "can add an ipld object using dag-cbor to dag-cbor" ' + IPLDCBORHASH=$(cat ipld_object_dagcbor | ipfs dag put --input-enc=dag-cbor -f dag-cbor) + ' + + test_expect_success "dag-cbor CID looks correct" ' + EXPHASH="bafyreibuwyi6b7oruz64krsakf5a4zmttu6dqksu6bpkpy3sjrm6a73o6a" + test $EXPHASH = $IPLDCBORHASH + ' + + test_expect_success "can add an ipld object using dag-cbor to dag-json" ' + IPLDJSONHASH=$(cat ipld_object_dagcbor | ipfs dag put --input-enc=dag-cbor -f dag-json) + ' + + test_expect_success "dag-cbor CID looks correct" ' + EXPHASH="baguqeerarvo2dwtckziergct46ijehc4yc7o3y7rwsiomwf65frbhqtgc4ta" + test $EXPHASH = $IPLDJSONHASH + ' + test_expect_success "various path traversals work" ' ipfs cat $IPLDHASH/cats/0 > out1 && ipfs cat $IPLDHASH/cats/1/water > out2 && @@ -81,20 +115,32 @@ test_dag_cmd() { ' test_expect_success "sub-objects look right" ' - echo "\"world\"" > sub1_exp && + echo -n "\"world\"" > sub1_exp && test_cmp sub1_exp sub1 && - echo "{\"beep\":[0,\"bop\"],\"dict\":\"ionary\"}" > sub2_exp && + echo -n "{\"beep\":[0,\"bop\"],\"dict\":\"ionary\"}" > sub2_exp && test_cmp sub2_exp sub2 && - echo "[0,\"bop\"]" > sub3_exp && + echo -n "[0,\"bop\"]" > sub3_exp && test_cmp sub3_exp sub3 && - echo "0" > sub4_exp && + echo -n "0" > sub4_exp && test_cmp sub4_exp sub4 && - echo "\"bop\"" > sub5_exp && + echo -n "\"bop\"" > sub5_exp && test_cmp sub5_exp sub5 ' - test_expect_success "can pin cbor object" ' - ipfs pin add $EXPHASH + test_expect_success "can pin ipld object" ' + ipfs pin add $IPLDHASH + ' + + test_expect_success "can pin dag-pb object" ' + ipfs pin add $IPLDPBHASH + ' + + test_expect_success "can pin dag-cbor object" ' + ipfs pin add $IPLDCBORHASH + ' + + test_expect_success "can pin dag-json object" ' + ipfs pin add $IPLDJSONHASH ' test_expect_success "after gc, objects still accessible" ' @@ -111,7 +157,7 @@ test_dag_cmd() { ' test_expect_success "retrieved object hashes back correctly" ' - IPLDHASH2=$(cat ipld_obj_out | ipfs dag put) && + IPLDHASH2=$(cat ipld_obj_out | ipfs dag put --input-enc=dag-json -f dag-cbor) && test "$IPLDHASH" = "$IPLDHASH2" ' @@ -124,7 +170,7 @@ test_dag_cmd() { ' test_expect_success "output looks correct" ' - echo "{\"data\":\"CAISB2Zvb2JhcgoYBw==\",\"links\":[]}" > dag_get_pb_exp && + echo -n "{\"Data\":{\"/\":{\"bytes\":\"CAISB2Zvb2JhcgoYBw==\"}},\"Links\":[]}" > dag_get_pb_exp && test_cmp dag_get_pb_exp dag_get_pb_out ' @@ -133,24 +179,23 @@ test_dag_cmd() { ' test_expect_success "output looks correct" ' - echo "{\"data\":\"CAISBGZvbwoYBA==\",\"links\":[]}" > cat_exp && + echo -n "{\"Data\":{\"/\":{\"bytes\":\"CAISBGZvbwoYBA==\"}},\"Links\":[]}" > cat_exp && test_cmp cat_exp cat_out ' - test_expect_success "non-canonical cbor input is normalized" ' - HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=cbor --input-enc=raw) && + test_expect_success "non-canonical dag-cbor input is normalized" ' + HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=dag-cbor --input-enc=dag-cbor) && test $HASH = "bafyreiawx7ona7oa2ptcoh6vwq4q6bmd7x2ibtkykld327bgb7t73ayrqm" || test_fsh echo $HASH ' - test_expect_success "non-canonical cbor input is normalized with input-enc cbor" ' - HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=cbor --input-enc=cbor) && - test $HASH = "bafyreiawx7ona7oa2ptcoh6vwq4q6bmd7x2ibtkykld327bgb7t73ayrqm" || - test_fsh echo $HASH + test_expect_success "cbor input can be fetched" ' + EXPARR=$(ipfs dag get $HASH/arr) + test $EXPARR = "[]" ' test_expect_success "add an ipld with pin" ' - PINHASH=$(printf {\"foo\":\"bar\"} | ipfs dag put --pin=true) + PINHASH=$(printf {\"foo\":\"bar\"} | ipfs dag put --input-enc=dag-json --pin=true) ' test_expect_success "after gc, objects still accessible" ' @@ -158,23 +203,23 @@ test_dag_cmd() { ipfs refs -r --timeout=2s $PINHASH > /dev/null ' - test_expect_success "can add an ipld object with sha3 hash" ' - IPLDHASH=$(cat ipld_object | ipfs dag put --hash sha3) + test_expect_success "can add an ipld object with sha3-512 hash" ' + IPLDHASH=$(cat ipld_object | ipfs dag put --hash sha3-512) ' test_expect_success "output looks correct" ' - EXPHASH="bafyriqgae54zjl3bjebmbat2rjem4ewj6vni6jxohandmvk3bibfgv3sioyeidppsghvulryxats43br3b7afa6jy77x6gqzqaicer6ljicck" + EXPHASH="bafyriqforjma7y7akqz7nhuu73r6liggj5zhkbfiqgicywe3fgkna2ijlhod2af3ue7doj56tlzt5hh6iu5esafc4msr3sd53jol5m2o25ucy" test $EXPHASH = $IPLDHASH ' test_expect_success "prepare dag-pb object" ' echo foo > test_file && - HASH=$(ipfs add -wq test_file | tail -n1) + HASH=$(ipfs add -wq test_file | tail -n1 | ipfs cid base32) ' test_expect_success "dag put with json dag-pb works" ' ipfs dag get $HASH > pbjson && - cat pbjson | ipfs dag put --format=dag-pb --input-enc=json > dag_put_out + cat pbjson | ipfs dag put --format=dag-pb --input-enc=dag-json > dag_put_out ' test_expect_success "dag put with dag-pb works output looks good" ' @@ -184,7 +229,7 @@ test_dag_cmd() { test_expect_success "dag put with raw dag-pb works" ' ipfs block get $HASH > pbraw && - cat pbraw | ipfs dag put --format=dag-pb --input-enc=raw > dag_put_out + cat pbraw | ipfs dag put --format=dag-pb --input-enc=dag-pb > dag_put_out ' test_expect_success "dag put with dag-pb works output looks good" ' @@ -271,10 +316,10 @@ test_dag_cmd() { test_expect_success "dag stat of simple IPLD object" ' ipfs dag stat $NESTED_HASH > actual_stat_inner_ipld_obj && - echo "Size: 15, NumBlocks: 1" > exp_stat_inner_ipld_obj && + echo "Size: 8, NumBlocks: 1" > exp_stat_inner_ipld_obj && test_cmp exp_stat_inner_ipld_obj actual_stat_inner_ipld_obj && ipfs dag stat $HASH > actual_stat_ipld_obj && - echo "Size: 61, NumBlocks: 2" > exp_stat_ipld_obj && + echo "Size: 54, NumBlocks: 2" > exp_stat_ipld_obj && test_cmp exp_stat_ipld_obj actual_stat_ipld_obj ' diff --git a/test/sharness/t0280-plugin-git.sh b/test/sharness/t0280-plugin-git.sh index df92e09f590..c9720a0dd6f 100755 --- a/test/sharness/t0280-plugin-git.sh +++ b/test/sharness/t0280-plugin-git.sh @@ -17,19 +17,19 @@ test_expect_success "prepare test data" ' test_dag_git() { test_expect_success "add objects via dag put" ' - find objects -type f -exec ipfs dag put --format=git --input-enc=zlib {} \; -exec echo \; > hashes + find objects -type f -exec ipfs dag put --format=git-raw --input-enc=0x300078 --hash=sha1 {} \; -exec echo -n \; > hashes ' - test_expect_failure "successfully get added objects" ' + test_expect_success "successfully get added objects" ' cat hashes | xargs -I {} ipfs dag get -- {} > /dev/null ' - test_expect_failure "path traversals work" ' - echo \"YmxvYiA3ACcsLnB5Zgo=\" > file1 && + test_expect_success "path traversals work" ' + echo -n "{\"/\":{\"bytes\":\"YmxvYiA3ACcsLnB5Zgo=\"}}" > file1 && ipfs dag get baf4bcfhzi72pcj5cc4ocz7igcduubuu7aa3cddi/object/parents/0/tree/dir2/hash/f3/hash > out1 ' - test_expect_failure "outputs look correct" ' + test_expect_success "outputs look correct" ' test_cmp file1 out1 ' }