diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index c1346059309..944c22bcec3 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -786,7 +786,7 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e var opts = []corehttp.ServeOption{ corehttp.MetricsCollectionOption("gateway"), corehttp.HostnameOption(), - corehttp.GatewayOption(writable, "/ipfs", "/ipns"), + corehttp.GatewayOption(writable, "/ipfs", "/ipns", "/ipld"), corehttp.VersionOption(), corehttp.CheckVersionOption(), corehttp.CommandsROOption(cmdctx), diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index 3d31abaabc2..262283aa87e 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -14,9 +14,15 @@ Interfaces here aren't yet completely stable. package coreapi import ( + "bytes" "context" "errors" "fmt" + ipld2 "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "io" bserv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-fetcher" @@ -91,6 +97,25 @@ func NewCoreAPI(n *core.IpfsNode, opts ...options.ApiOption) (coreiface.CoreAPI, return (&CoreAPI{nd: n, parentOpts: *parentOpts}).WithOptions(opts...) } +var KnownReifiers map[string]ipld2.NodeReifier = make(map[string]ipld2.NodeReifier) + +func (api *CoreAPI) LinkSystem() ipld2.LinkSystem { + lsys := cidlink.DefaultLinkSystem() + lsys.KnownReifiers = KnownReifiers + lsys.StorageReadOpener = func(linkContext linking.LinkContext, link datamodel.Link) (io.Reader, error) { + if cl, ok := link.(cidlink.Link); !ok { + return nil, fmt.Errorf("cannot process link: %v", link) + } else { + block, err := api.blocks.GetBlock(linkContext.Ctx, cl.Cid) + if err != nil { + return nil, err + } + return bytes.NewReader(block.RawData()), nil + } + } + return lsys +} + // Unixfs returns the UnixfsAPI interface implementation backed by the go-ipfs node func (api *CoreAPI) Unixfs() coreiface.UnixfsAPI { return (*UnixfsAPI)(api) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index f32fac54ecd..a60544b5e64 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -3,6 +3,12 @@ package corehttp import ( "context" "fmt" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-multihash" "html/template" "io" "mime" @@ -364,22 +370,35 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request return } - // Resolve path to the final DAG node for the ETag - resolvedPath, err := i.api.ResolvePath(r.Context(), contentPath) - switch err { - case nil: - case coreiface.ErrOffline: - webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusServiceUnavailable) - return - default: - // if Accept is text/html, see if ipfs-404.html is present - if i.servePretty404IfPresent(w, r, contentPath) { - logger.Debugw("serve pretty 404 if present") + ns := contentPath.Namespace() + + var resolvedPath ipath.Resolved + var err error + if ns != "ipld" { + // Resolve path to the final DAG node for the ETag + resolvedPath, err = i.api.ResolvePath(r.Context(), contentPath) + switch err { + case nil: + case coreiface.ErrOffline: + webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusServiceUnavailable) return - } + default: + // if Accept is text/html, see if ipfs-404.html is present + if i.servePretty404IfPresent(w, r, contentPath) { + logger.Debugw("serve pretty 404 if present") + return + } - webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusNotFound) - return + webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusNotFound) + return + } + } else { + cstr := strings.Split(contentPath.String(), "/")[2] + c, err := cid.Decode(cstr) + if err != nil { + webError(w, "resolve /ipld error "+debugStr(contentPath.String()), err, http.StatusNotFound) + } + resolvedPath = ipath.IpfsPath(c) } // Detect when explicit Accept header or ?format parameter are present @@ -415,11 +434,95 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request return } + var querySelector ipld.Node + var selectorCID cid.Cid + if selectorParam := r.URL.Query().Get("selector"); selectorParam != "" { + selectorCID, err = cid.Decode(selectorParam) + if err != nil { + webError(w, "selector is not a valid CID", err, http.StatusInternalServerError) + return + } + selectorNode, err := i.api.Dag().Get(r.Context(), selectorCID) + if err != nil { + webError(w, "could not get selector", err, http.StatusInternalServerError) + return + } + querySelector = selectorNode.(ipld.Node) + } + + if ipldPathParam := r.URL.Query().Get("ipld-path"); ipldPathParam != "" || ns == "ipld" { + var pcs []string + if ipldPathParam != "" { + pcs = strings.Split(ipldPathParam, "|") + } else { + pcs = strings.Split(contentPath.String(), "/")[3:] + } + buildSel := builder.NewSelectorSpecBuilder(basicnode.Prototype__Any{}) + var fnBuildSel func(comps []string) (builder.SelectorSpec, error) + fnBuildSel = func(comps []string) (builder.SelectorSpec, error) { + if len(comps) == 0 { + return buildSel.Matcher(), nil + } + comp := comps[0] + if comp[0] != '[' { + next, err := fnBuildSel(comps[1:]) + if err != nil { + return nil, err + } + return buildSel.ExploreFields(func(specBuilder builder.ExploreFieldsSpecBuilder) { + specBuilder.Insert(comp, next) + }), nil + } else { + matched, err := regexp.MatchString(`[ADL=(.*)]`, comp) + if err != nil { + return nil, err + } + if !matched { + return nil, fmt.Errorf("invalid path component %s", comp) + } + adlName := comp[5 : len(comp)-1] + next, err := fnBuildSel(comps[1:]) + if err != nil { + return nil, err + } + return buildSel.ExploreInterpretAs(adlName, next), nil + } + } + + sel, err := fnBuildSel(pcs) + if err != nil { + webError(w, "problem with ipld pathing", err, http.StatusBadRequest) + return + } + + querySelector = sel.Node() + lsys := cidlink.DefaultLinkSystem() + lnk, err := lsys.ComputeLink(cidlink.LinkPrototype{Prefix: cid.Prefix{ + Version: 1, + Codec: uint64(multicodec.DagJson), + MhType: multihash.IDENTITY, + MhLength: -1, + }}, querySelector) + + selectorCIDLnk, ok := lnk.(cidlink.Link) + if !ok { + webError(w, "could not compute selector CID", err, http.StatusInternalServerError) + return + } + selectorCID = selectorCIDLnk.Cid + } + // Support custom response formats passed via ?format or Accept HTTP header switch responseFormat { case "": // The implicit response format is UnixFS - logger.Debugw("serving unixfs", "path", contentPath) - i.serveUnixFS(r.Context(), w, r, resolvedPath, contentPath, begin, logger) + // If there is a selector we're in IPLD land + if querySelector != nil { + logger.Debugw("serving ipld selector request", "path", contentPath, "selector", selectorCID) + i.serveIPLD(w, r, resolvedPath, contentPath, querySelector, selectorCID, begin, logger) + } else { + logger.Debugw("serving unixfs", "path", contentPath) + i.serveUnixFS(r.Context(), w, r, resolvedPath, contentPath, begin, logger) + } return case "application/vnd.ipld.raw": logger.Debugw("serving raw block", "path", contentPath) @@ -1071,11 +1174,12 @@ func (i *gatewayHandler) setCommonHeaders(w http.ResponseWriter, r *http.Request i.addUserHeaders(w) // ok, _now_ write user's headers. w.Header().Set("X-Ipfs-Path", contentPath.String()) - if rootCids, err := i.buildIpfsRootsHeader(contentPath.String(), r); err == nil { - w.Header().Set("X-Ipfs-Roots", rootCids) - } else { // this should never happen, as we resolved the contentPath already - return newRequestError("error while resolving X-Ipfs-Roots", err, http.StatusInternalServerError) + if contentPath.Namespace() != "ipld" { + if rootCids, err := i.buildIpfsRootsHeader(contentPath.String(), r); err == nil { + w.Header().Set("X-Ipfs-Roots", rootCids) + } else { // this should never happen, as we resolved the contentPath already + return newRequestError("error while resolving X-Ipfs-Roots", err, http.StatusInternalServerError) + } } - return nil } diff --git a/core/corehttp/gateway_handler_selector.go b/core/corehttp/gateway_handler_selector.go new file mode 100644 index 00000000000..726c0260e12 --- /dev/null +++ b/core/corehttp/gateway_handler_selector.go @@ -0,0 +1,107 @@ +package corehttp + +import ( + "fmt" + "github.com/ipfs/go-cid" + "net/http" + "time" + + "go.uber.org/zap" + + ipath "github.com/ipfs/interface-go-ipfs-core/path" + + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" +) + +func (i *gatewayHandler) serveIPLD(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, selectorNode ipld.Node, selectorCID cid.Cid, begin time.Time, logger *zap.SugaredLogger) { + if resolvedPath.Remainder() != "" { + http.Error(w, "serving ipld cannot handle path remainders", http.StatusInternalServerError) + return + } + + nsc := func(lnk ipld.Link, lctx ipld.LinkContext) (ipld.NodePrototype, error) { + // We can decode all nodes into basicnode's Any, except for + // dagpb nodes, which must explicitly use the PBNode prototype. + if lnk, ok := lnk.(cidlink.Link); ok && lnk.Cid.Prefix().Codec == 0x70 { + return dagpb.Type.PBNode, nil + } + return basicnode.Prototype.Any, nil + } + + compiledSelector, err := selector.CompileSelector(selectorNode) + if err != nil { + webError(w, "could not compile selector", err, http.StatusInternalServerError) + return + } + + lnk := cidlink.Link{Cid: resolvedPath.Cid()} + ns, _ := nsc(lnk, ipld.LinkContext{}) // nsc won't error + + type HasLinksystem interface { + LinkSystem() ipld.LinkSystem + } + + lsHaver, ok := i.api.(HasLinksystem) + if !ok { + webError(w, "could not find linksystem", err, http.StatusInternalServerError) + return + } + lsys := lsHaver.LinkSystem() + + nd, err := lsys.Load(ipld.LinkContext{Ctx: r.Context()}, lnk, ns) + if err != nil { + webError(w, "could not load root", err, http.StatusInternalServerError) + return + } + + prog := traversal.Progress{ + Cfg: &traversal.Config{ + Ctx: r.Context(), + LinkSystem: lsys, + LinkTargetNodePrototypeChooser: nsc, + LinkVisitOnlyOnce: true, + }, + } + + var latestMatchedNode ipld.Node + + err = prog.WalkAdv(nd, compiledSelector, func(progress traversal.Progress, node datamodel.Node, reason traversal.VisitReason) error { + if reason == traversal.VisitReason_SelectionMatch { + if latestMatchedNode == nil { + latestMatchedNode = node + } else { + return fmt.Errorf("can only use selectors that match a single node") + } + } + return nil + }) + if err != nil { + webError(w, "could not execute selector", err, http.StatusInternalServerError) + return + } + + if latestMatchedNode == nil { + webError(w, "selector did not match anything", err, http.StatusInternalServerError) + return + } + + lbnNode, ok := latestMatchedNode.(datamodel.LargeBytesNode) + if !ok { + webError(w, "matched node was not bytes", err, http.StatusInternalServerError) + return + } + if data, err := lbnNode.AsLargeBytes(); err != nil { + webError(w, "matched node was not bytes", err, http.StatusInternalServerError) + return + } else { + modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) + name := resolvedPath.Cid().String() + "-" + selectorCID.String() + http.ServeContent(w, r, name, modtime, data) + } +} diff --git a/core/corehttp/hostname.go b/core/corehttp/hostname.go index 93dde67ab28..7ed5ba7f7b1 100644 --- a/core/corehttp/hostname.go +++ b/core/corehttp/hostname.go @@ -24,7 +24,7 @@ import ( nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" ) -var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"} +var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/", "/ipld/"} var subdomainGatewaySpec = &config.GatewaySpec{ Paths: defaultPaths, diff --git a/go.mod b/go.mod index bcdd9069ec7..95f6b3c10cb 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,9 @@ module github.com/ipfs/go-ipfs require ( - bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc + bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512 contrib.go.opencensus.io/exporter/prometheus v0.4.0 + github.com/aschmahmann/wasm-ipld/gobind v0.0.0-20220607151816-3afb0185f645 github.com/blang/semver/v4 v4.0.0 github.com/cenkalti/backoff/v4 v4.1.3 github.com/ceramicnetwork/go-dag-jose v0.1.0 @@ -96,7 +97,7 @@ require ( github.com/multiformats/go-multiaddr v0.5.0 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multibase v0.0.3 - github.com/multiformats/go-multicodec v0.4.1 + github.com/multiformats/go-multicodec v0.5.0 github.com/multiformats/go-multihash v0.1.0 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 @@ -128,6 +129,7 @@ require ( require ( github.com/benbjohnson/clock v1.3.0 github.com/ipfs/go-log/v2 v2.5.1 + github.com/mitchellh/mapstructure v1.1.2 ) require ( @@ -138,6 +140,7 @@ require ( github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/btcsuite/btcd v0.22.1 // indirect + github.com/bytecodealliance/wasmtime-go v0.36.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cheekybits/genny v1.0.0 // indirect @@ -165,6 +168,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect @@ -225,6 +229,7 @@ require ( github.com/multiformats/go-varint v0.0.6 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/gomega v1.17.0 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/openzipkin/zipkin-go v0.4.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect @@ -235,6 +240,7 @@ require ( github.com/prometheus/statsd_exporter v0.21.0 // indirect github.com/raulk/clock v1.1.0 // indirect github.com/raulk/go-watchdog v1.2.0 // indirect + github.com/rogpeppe/go-internal v1.8.0 // indirect github.com/rs/cors v1.7.0 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 09757be3eb4..3a139bb953e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc h1:utDghgcjE8u+EBjHOgYT+dJPcnDF05KqWMBcjuJy510= -bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= +bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512 h1:SRsZGA7aFnCZETmov57jwPrWuTmaZK6+4R4v5FUe1/c= +bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -81,6 +81,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aschmahmann/wasm-ipld/gobind v0.0.0-20220607151816-3afb0185f645 h1:XGPLZbU0IRs4BBCOuaX1oBd1XA/Ikx3+UKf7P3za7Tg= +github.com/aschmahmann/wasm-ipld/gobind v0.0.0-20220607151816-3afb0185f645/go.mod h1:wMtlN3l6ULwWNYUjBLqXzWE3BkPM1IuyUWeq+iZhwPA= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= @@ -121,6 +123,8 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bytecodealliance/wasmtime-go v0.36.0 h1:B6thr7RMM9xQmouBtUqm1RpkJjuLS37m6nxX+iwsQSc= +github.com/bytecodealliance/wasmtime-go v0.36.0/go.mod h1:q320gUxqyI8yB+ZqRuaJOEnGkAnHh6WtJjMaT2CW4wI= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -377,8 +381,9 @@ github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE0 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -1205,6 +1210,7 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1261,8 +1267,9 @@ github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErh github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.4.1 h1:BSJbf+zpghcZMZrwTYBGwy0CPcVZGWiC72Cp8bBd4R4= github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.5.0 h1:EgU6cBe/D7WRwQb1KmnBvU7lrcFGMggZVTPtOW9dDHs= +github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1322,8 +1329,9 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= -github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1352,6 +1360,7 @@ github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2u github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1421,8 +1430,9 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqn github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -1914,6 +1924,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025112917-711f33c9992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/plugin/ipld.go b/plugin/ipld.go index 2f783a61238..10ae819ba68 100644 --- a/plugin/ipld.go +++ b/plugin/ipld.go @@ -1,6 +1,7 @@ package plugin import ( + "github.com/ipld/go-ipld-prime" multicodec "github.com/ipld/go-ipld-prime/multicodec" ) @@ -11,3 +12,9 @@ type PluginIPLD interface { Register(multicodec.Registry) error } + +type PluginIPLDADL interface { + Plugin + + RegisterADL(map[string]ipld.NodeReifier) error +} diff --git a/plugin/loader/loader.go b/plugin/loader/loader.go index 3c52a4105ad..6e0044d5c5a 100644 --- a/plugin/loader/loader.go +++ b/plugin/loader/loader.go @@ -248,6 +248,15 @@ func (loader *PluginLoader) Inject() error { return err } } + + if pl, ok := pl.(plugin.PluginIPLDADL); ok { + err := injectIPLDADLPlugin(pl) + if err != nil { + loader.state = loaderFailed + return err + } + } + if pl, ok := pl.(plugin.PluginTracer); ok { err := injectTracerPlugin(pl) if err != nil { @@ -338,6 +347,10 @@ func injectIPLDPlugin(pl plugin.PluginIPLD) error { return pl.Register(multicodec.DefaultRegistry) } +func injectIPLDADLPlugin(pl plugin.PluginIPLDADL) error { + return pl.RegisterADL(coreapi.KnownReifiers) +} + func injectTracerPlugin(pl plugin.PluginTracer) error { log.Warn("Tracer plugins are deprecated, it's recommended to configure an OpenTelemetry collector instead.") tracer, err := pl.InitTracer() diff --git a/plugin/loader/preload.go b/plugin/loader/preload.go index 17da3c2f7ad..a52515f2ee6 100644 --- a/plugin/loader/preload.go +++ b/plugin/loader/preload.go @@ -7,6 +7,7 @@ import ( pluginipldgit "github.com/ipfs/go-ipfs/plugin/plugins/git" pluginlevelds "github.com/ipfs/go-ipfs/plugin/plugins/levelds" pluginpeerlog "github.com/ipfs/go-ipfs/plugin/plugins/peerlog" + pluginipldwasm "github.com/ipfs/go-ipfs/plugin/plugins/wasmipld" ) // DO NOT EDIT THIS FILE @@ -16,6 +17,7 @@ import ( func init() { Preload(pluginipldgit.Plugins...) Preload(pluginiplddagjose.Plugins...) + Preload(pluginipldwasm.Plugins...) Preload(pluginbadgerds.Plugins...) Preload(pluginflatfs.Plugins...) Preload(pluginlevelds.Plugins...) diff --git a/plugin/plugins/wasmipld/spec.md b/plugin/plugins/wasmipld/spec.md new file mode 100644 index 00000000000..7e541a58b60 --- /dev/null +++ b/plugin/plugins/wasmipld/spec.md @@ -0,0 +1,209 @@ +--- +title: "WAC Specification" +navTitle: "Spec" +--- + +# Specification: WAC + +**Status: Prescriptive - Exploratory ** + +* [Format](#format) +* [Links](#links) +* [Map Keys](#map-keys) +* [Strictness](#strictness) +* [Implementations](#implementations) + * [JavaScript](#javascript) + * [Go](#go) + * [Java](#java) +* [Limitations](#limitations) + * [JavaScript Numbers](#javascript-numbers) + +WAC supports bidirectional transport to and from the [IPLD Data Model]. +In that way it is both a "complete" IPLD Data Model representation and "fitted" to the IPLD Data Model. +Terminology taken from [Codecs and Completeness]. + +It takes some inspiration from [Simple DAG], but differs in ways designed to make it bidrectional with the IPLD Data Model. + +## Format + +# Spec + +This format is a series of typing tokens, constant tokens, and inline value data. + +Typeing tokens are proceeded with the value data for that type. + +Every type value can be parsed knowing only the type and without any outside context +like the container or positional delimiters. + +Tokens + +## TODO: Make constants in code match spec + +| Int | Token | +|---|---| +| 0 | TYPE_LINK | +| 1 | TYPE_INTEGER | +| 2 | TYPE_NEGATIVE_INTEGER | +| 3 | TYPE_FLOAT | +| 4 | TYPE_NEGATIVE_FLOAT | +| 5 | TYPE_STRING | +| 6 | TYPE_BINARY | +| 7 | TYPE_MAP | +| 8 | TYPE_LIST | +| 9 | VALUE_NULL | +| 10 | VALUE_TRUE | +| 11 | VALUE_FALSE | + +## TYPE_LINK + +``` +| 0 | CID | +``` + +Note: Simple-DAG put a VARINT_LENGTH before the CID indicating how long it was. + +CIDs are self-delimiting so this didn't seem necessary. + +## TYPE_INTEGER + +``` +| 1 | VARINT | +``` + +## TYPE_SIGNED_INTEGER + +``` +| 2 | VARINT | +``` + +## TYPE_FLOAT + +``` +| 3 | MATISSA_LENGTH | VARINT +``` + +TODO: Floats (and negative floats) need definition here. +Making implementations actually bidirectional with respect to the IPLD Data Model seems difficult here. + +## TYPE_NEGATIVE_FLOAT + +``` +| 4 | MATISSA_LENGTH | VARINT +``` + +## TYPE_STRING + +``` +| 5 | VARINT_LENGTH | STRING +``` + +Note: This is essentially the same as Binary, but with a different token flag + +## TYPE_BINARY + +``` +| 6 | VARINT_LENGTH | BINARY +``` + +## TYPE_MAP + +``` +| 7 | VARINT_NUM_PAIRS | PAIRS +``` + +`PAIRS` contains alternating keys then values concatenated. i.e. ` KEY1 | VALUE1 | KEY2 | VALUE2 ...` + +This codec does not have any form of canonical map sorting as that would make it ill-fitted to the IPLD Data Model. + +As in the IPLD Data Model map keys must be of type String, however as described in the String section the only +distinction between Strings and Bytes are identifier hints. + +Note: Simple-DAG went with `KEYS_VARINT_LENGTH | VALUES_VARINT_LENGTH | KEYS | VALUES |`. Both seem doable, +this approach seemed to make writing encoder/decoders really simple. However, adding in more length prefixes +makes creating faster "zero copy" decoders very nice. It seems to mostly be a tradeoff for which side has to have bigger +buffers, the encoder or the decoder. + +Note: We could assert that keys are just `VARINT_LENGTH | STRING` and remove the String token since it's always a string. +It's some added complexity and really stops us from putting anything other than Strings in map keys, but that may not be +too bad. + +## TYPE_LIST + +``` +| 8 | VARINT_NUM_ELEMENTS | VALUES | +``` + +`VALUES` contains every value concatenated. + +Note: Simple-DAG went with `VARINT_LENGTH` (the size of the VALUES binary section) instead of `VARINT_NUM_ELEMENTS` +and in the `VALUES` section had every value proceeded by the length of the value. + +Both seem doable, this approach seemed to make writing a decoder really simple. However, adding in more length prefixes +makes creating faster "zero copy" decoders very nice. It seems to mostly be a tradeoff for which side has to have bigger +buffers, the encoder or the decoder. + +## TYPE_NULL + +``` +| 9 | +``` + +## TYPE_TRUE + +``` +| 10 | +``` + +## TYPE_FALSE + +``` +| 11 | +``` + +## Strings + + + +## Strictness + + + +## Implementations + +### JavaScript + +**[@ipld/dag-cbor]**, for use with [multiformats] adheres to this specification, with the following caveats: +* Complete strictness is not yet enforced on decode. Specifically: correct map ordering is not enforced and floats that are not encoded as 64-bit are not rejected. +* [`BigInt`] is accepted along with `Number` for encode, but the smallest-possible rule is followed when encoding. When decoding integers outside of the JavaScript "safe integer" range, a [`BigInt`] will be used. + +The legacy **[ipld-dag-cbor]** implementation adheres to this specification, with the following caveats: + +* Strictness is not enforced on decode; blocks encoded that do not follow the strictness rules are not rejected. +* Floating point values are encoded as their smallest form rather than always 64-bit. +* Many additional object types outside of the Data Model are currently accepted for decode and encode, including `undefined`. +* [IEEE 754] special values `NaN`, `Infinity` and `-Infinity` are accepted for decode and encode. +* Integers outside of the JavaScript "safe integer" range will use the third-party [bignumber.js] library to represent their values. + +Note that inability to clearly differentiate between integers and floats in JavaScript may cause problems with round-trips of floating point values. See the [IPLD Data Model] and the discussion on [Limitations](#limitations) below for further discussion on JavaScript numbers and recommendations regarding the use of floats. + +### Go + +Here and adheres to the specification. However, in a practical sense because it implements the go-ipld-prime interface +its limits on integers and floats are currently bounded by those interfaces. Similarly any varints are capped around 64 bits. + +### Rust/WASM + +Adheres to the specification. However, in a practical sense its limits on integers and floats are currently bounded to be +of fixed maximum sizes. Similarly, all varints are capped around 64 bits. + +## Limitations + +[IPLD Data Model]: /docs/data-model/ +[Concise Binary Object Representation (CBOR)]: https://cbor.io/ +[IPLD Data Model Kinds]: /docs/data-model/kinds/ +[Links]: /docs/data-model/kinds/#link-kind +[CID]: /glossary/#cid +[Multibase]: https://github.com/multiformats/multibase +[go-ipld-prime]: http://github.com/ipld/go-ipld-prime +[Codecs and Completeness] : https://gist.github.com/warpfork/28f93bee7184a708223274583109f31c +[Simple DAG] : https://github.com/mikeal/simple-dag \ No newline at end of file diff --git a/plugin/plugins/wasmipld/wasmipld.go b/plugin/plugins/wasmipld/wasmipld.go new file mode 100644 index 00000000000..35abf14a1d9 --- /dev/null +++ b/plugin/plugins/wasmipld/wasmipld.go @@ -0,0 +1,153 @@ +package wasmipld + +import ( + "io/ioutil" + + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/multicodec" + + mc "github.com/multiformats/go-multicodec" + + "github.com/ipfs/go-ipfs/plugin" + "github.com/mitchellh/mapstructure" + + wasmbind "github.com/aschmahmann/wasm-ipld/gobind" +) + +// Plugins is exported list of plugins that will be loaded +var Plugins = []plugin.Plugin{ + &wasmipld{}, +} + +type wasmipld struct{} + +var _ plugin.PluginIPLD = (*wasmipld)(nil) +var _ plugin.PluginIPLDADL = (*wasmipld)(nil) + +func (*wasmipld) Name() string { + return "ipld-wasmipld" +} + +func (*wasmipld) Version() string { + return "0.0.1" +} + +type WasmIPLDConfig struct { + Codecs []WasmIPLDCodecConfig + ADLs []WasmIPLDADLConfig +} + +type WasmIPLDCodecConfig struct { + Code string + Encode bool + Decode bool + WasmPath string +} + +type WasmIPLDADLConfig struct { + Name string + WasmPath string +} + +func (*wasmipld) Init(env *plugin.Environment) error { + config := env.Config + if config == nil { + return nil + } + + var cfg WasmIPLDConfig + // Note: This dependency is just for convenience and is in our go.sum anyway + // It can go away if we upstream this into the main config instead of keeping it as a plugin + if err := mapstructure.Decode(config, &cfg); err != nil { + return err + } + + registry = &wasmRegistry{} + + for _, c := range cfg.Codecs { + wasm, err := ioutil.ReadFile(c.WasmPath) + if err != nil { + return err + } + + var code mc.Code + if err := code.Set(c.Code); err != nil { + return err + } + + codecRegistration := codecReg{ + code: code, + wasm: wasm, + encode: c.Encode, + decode: c.Decode, + } + registry.Codecs = append(registry.Codecs, codecRegistration) + } + + for _, a := range cfg.ADLs { + wasm, err := ioutil.ReadFile(a.WasmPath) + if err != nil { + return err + } + + adlRegistration := adlReg{ + name: a.Name, + wasm: wasm, + } + + registry.ADLs = append(registry.ADLs, adlRegistration) + } + + return nil +} + +const fuelPerOp = 10_000_000 + +var registry *wasmRegistry + +type wasmRegistry struct { + Codecs []codecReg + ADLs []adlReg +} + +type codecReg struct { + code mc.Code + wasm []byte + encode bool + decode bool +} + +type adlReg struct { + name string + wasm []byte +} + +func (*wasmipld) Register(reg multicodec.Registry) error { + reg.RegisterEncoder(wasmbind.WacMC, wasmbind.WacEncode) + reg.RegisterDecoder(wasmbind.WacMC, wasmbind.WacDecode) + + for _, c := range registry.Codecs { + codec, err := wasmbind.NewWasmCodec(c.wasm, wasmbind.WasmCodecOptions{}.WithFuelPerOp(fuelPerOp)) + if err != nil { + return err + } + if c.encode { + reg.RegisterEncoder(uint64(c.code), codec.Encode) + } + if c.decode { + reg.RegisterDecoder(uint64(c.code), codec.Decode) + } + } + return nil +} + +func (b *wasmipld) RegisterADL(m map[string]ipld.NodeReifier) error { + for _, a := range registry.ADLs { + adl, err := wasmbind.NewWasmADL(a.wasm, wasmbind.WasmADLOptions{}.WithFuelPerOp(fuelPerOp)) + if err != nil { + return err + } + m[a.name] = adl.Reify + } + return nil +}