diff --git a/Rules.mk b/Rules.mk index 441e7ec4a61..ca9d1bf25d7 100644 --- a/Rules.mk +++ b/Rules.mk @@ -27,6 +27,10 @@ export IPFS_REUSEPORT=false dir := bin include $(dir)/Rules.mk +# tests need access to rules from plugin +dir := plugin +include $(dir)/Rules.mk + dir := test include $(dir)/Rules.mk @@ -56,6 +60,7 @@ include $(dir)/Rules.mk dir := pin/internal/pb include $(dir)/Rules.mk + # -------------------- # # universal rules # # -------------------- # @@ -142,7 +147,7 @@ help: @echo ' test_go_short' @echo ' test_go_expensive' @echo ' test_go_race' - @echo ' test_go_megacheck' - Run the `megacheck` vetting tool + @echo ' test_go_megacheck - Run the `megacheck` vetting tool' @echo ' test_sharness_short' @echo ' test_sharness_expensive' @echo ' test_sharness_race' diff --git a/cmd/ipfs/main.go b/cmd/ipfs/main.go index f555e0e1297..dcb2df40701 100644 --- a/cmd/ipfs/main.go +++ b/cmd/ipfs/main.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "os/signal" + "path/filepath" "runtime/pprof" "strings" "sync" @@ -22,6 +23,7 @@ import ( cmdsHttp "github.com/ipfs/go-ipfs/commands/http" core "github.com/ipfs/go-ipfs/core" coreCmds "github.com/ipfs/go-ipfs/core/commands" + "github.com/ipfs/go-ipfs/plugin/loader" repo "github.com/ipfs/go-ipfs/repo" config "github.com/ipfs/go-ipfs/repo/config" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" @@ -339,6 +341,11 @@ func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd } else { log.Debug("executing command locally") + pluginpath := filepath.Join(req.InvocContext().ConfigRoot, "plugins") + if _, err := loader.LoadPlugins(pluginpath); err != nil { + return nil, err + } + err := req.SetRootContext(ctx) if err != nil { return nil, err diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 842ec3574bd..f61f3a1e564 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -7,6 +7,7 @@ import ( "strings" cmds "github.com/ipfs/go-ipfs/commands" + coredag "github.com/ipfs/go-ipfs/core/coredag" path "github.com/ipfs/go-ipfs/path" pin "github.com/ipfs/go-ipfs/pin" @@ -76,34 +77,25 @@ into an object of the specified format. defer n.Blockstore.PinLock().Unlock() } - var c *cid.Cid - switch ienc { - case "json": - nd, err := convertJsonToType(fi, format) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } + nds, err := coredag.ParseInputs(ienc, format, fi) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } - c, err = n.DAG.Add(nd) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - case "raw": - nd, err := convertRawToType(fi, format) + var c *cid.Cid + b := n.DAG.Batch() + for _, nd := range nds { + cid, err := b.Add(nd) if err != nil { res.SetError(err, cmds.ErrNormal) return } - c, err = n.DAG.Add(nd) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - default: - res.SetError(fmt.Errorf("unrecognized input encoding: %s", ienc), cmds.ErrNormal) + c = cid + } + if err := b.Commit(); err != nil { + res.SetError(err, cmds.ErrNormal) return } diff --git a/core/coredag/dagtransl.go b/core/coredag/dagtransl.go new file mode 100644 index 00000000000..5e0f5d808a9 --- /dev/null +++ b/core/coredag/dagtransl.go @@ -0,0 +1,91 @@ +package coredag + +import ( + "fmt" + "io" + "io/ioutil" + + node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format" + ipldcbor "gx/ipfs/QmemYymP73eVdTUUMZEiSpiHeZQKNJdT5dP2iuHssZh1sR/go-ipld-cbor" +) + +// DagParser is function used for parsing stream into Node +type DagParser func(r io.Reader) ([]node.Node, error) + +// FormatParsers is used for mapping format descriptors to DagParsers +type FormatParsers map[string]DagParser + +// InputEncParsers is used for mapping input encodings to FormatParsers +type InputEncParsers map[string]FormatParsers + +// DefaultInputEncParsers is InputEncParser that is used everywhere +var DefaultInputEncParsers = InputEncParsers{ + "json": defaultJSONParsers, + "raw": defaultRawParsers, +} + +var defaultJSONParsers = FormatParsers{ + "cbor": cborJSONParser, + "dag-cbor": cborJSONParser, +} + +var defaultRawParsers = FormatParsers{ + "cbor": cborRawParser, + "dag-cbor": cborRawParser, +} + +// ParseInputs uses DefaultInputEncParsers to parse io.Reader described by +// input encoding and format to an instance of ipld Node +func ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) { + return DefaultInputEncParsers.ParseInputs(ienc, format, r) +} + +// AddParser adds DagParser under give input encoding and format +func (iep InputEncParsers) AddParser(ienv, format string, f DagParser) { + m, ok := iep[ienv] + if !ok { + m = make(FormatParsers) + iep[ienv] = m + } + + m[format] = f +} + +// ParseInputs parses io.Reader described by input encoding and format to +// an instance of ipld Node +func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) { + pset, ok := iep[ienc] + if !ok { + return nil, fmt.Errorf("no input parser for %q", ienc) + } + + parser, ok := pset[format] + if !ok { + return nil, fmt.Errorf("no parser for format %q using input type %q", format, ienc) + } + + return parser(r) +} + +func cborJSONParser(r io.Reader) ([]node.Node, error) { + nd, err := ipldcbor.FromJson(r) + if err != nil { + return nil, err + } + + return []node.Node{nd}, nil +} + +func cborRawParser(r io.Reader) ([]node.Node, error) { + data, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + nd, err := ipldcbor.Decode(data) + if err != nil { + return nil, err + } + + return []node.Node{nd}, nil +} diff --git a/coverage/Rules.mk b/coverage/Rules.mk index bf433621c0a..f137b18c9ad 100644 --- a/coverage/Rules.mk +++ b/coverage/Rules.mk @@ -1,6 +1,6 @@ include mk/header.mk -$(d)/coverage_deps: +$(d)/coverage_deps: $$(DEPS_GO) rm -rf $(@D)/unitcover && mkdir $(@D)/unitcover rm -rf $(@D)/sharnesscover && mkdir $(@D)/sharnesscover ifneq ($(IPFS_SKIP_COVER_BINS),1) @@ -41,6 +41,7 @@ endif export IPFS_COVER_DIR:= $(realpath $(d))/sharnesscover/ +$(d)/sharness_tests.coverprofile: export TEST_NO_PLUGIN=1 $(d)/sharness_tests.coverprofile: $(d)/ipfs cmd/ipfs/ipfs-test-cover $(d)/coverage_deps test_sharness_short (cd $(@D)/sharnesscover && find . -type f | gocovmerge -list -) > $@ diff --git a/mk/util.mk b/mk/util.mk index 05e2dbe7521..67b94f9520c 100644 --- a/mk/util.mk +++ b/mk/util.mk @@ -1,4 +1,5 @@ # util functions +OS ?= $(shell sh -c 'uname -s 2>/dev/null || echo not') ifeq ($(OS),Windows_NT) WINDOWS :=1 ?exe :=.exe # windows compat diff --git a/package.json b/package.json index bfc39d027bf..8f0e7a3514f 100644 --- a/package.json +++ b/package.json @@ -441,6 +441,12 @@ "hash": "QmPjTrrSfE6TzLv6ya6VWhGcCgPrUAdcgrDcQyRDX2VyW1", "name": "go-libp2p-routing", "version": "2.2.17" + }, + { + "author": "whyrusleeping", + "hash": "Qma7Kuwun7w8SZphjEPDVxvGfetBkqdNGmigDA13sJdLex", + "name": "go-ipld-git", + "version": "0.1.3" } ], "gxVersion": "0.10.0", diff --git a/plugin/Rules.mk b/plugin/Rules.mk new file mode 100644 index 00000000000..1e26d2a3c69 --- /dev/null +++ b/plugin/Rules.mk @@ -0,0 +1,9 @@ +include mk/header.mk + +dir := $(d)/loader +include $(dir)/Rules.mk + +dir := $(d)/plugins +include $(dir)/Rules.mk + +include mk/footer.mk diff --git a/plugin/ipld.go b/plugin/ipld.go new file mode 100644 index 00000000000..3dfdc0e04ca --- /dev/null +++ b/plugin/ipld.go @@ -0,0 +1,16 @@ +package plugin + +import ( + "github.com/ipfs/go-ipfs/core/coredag" + + node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format" +) + +// PluginIPLD is an interface that can be implemented to add handlers for +// for different IPLD formats +type PluginIPLD interface { + Plugin + + RegisterBlockDecoders(dec node.BlockDecoder) error + RegisterInputEncParsers(iec coredag.InputEncParsers) error +} diff --git a/plugin/loader/.gitignore b/plugin/loader/.gitignore new file mode 100644 index 00000000000..5f035563852 --- /dev/null +++ b/plugin/loader/.gitignore @@ -0,0 +1 @@ +preload.go diff --git a/plugin/loader/Rules.mk b/plugin/loader/Rules.mk new file mode 100644 index 00000000000..50e4e743f40 --- /dev/null +++ b/plugin/loader/Rules.mk @@ -0,0 +1,10 @@ +include mk/header.mk + +$(d)/preload.go: d:=$(d) +$(d)/preload.go: $(d)/preload_list + $(d)/preload.sh > $@ + go fmt $@ >/dev/null + +DEPS_GO += $(d)/preload.go + +include mk/footer.mk diff --git a/plugin/loader/initializer.go b/plugin/loader/initializer.go new file mode 100644 index 00000000000..7ad627eefa9 --- /dev/null +++ b/plugin/loader/initializer.go @@ -0,0 +1,43 @@ +package loader + +import ( + "github.com/ipfs/go-ipfs/core/coredag" + "github.com/ipfs/go-ipfs/plugin" + + format "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format" +) + +func initialize(plugins []plugin.Plugin) error { + for _, p := range plugins { + err := p.Init() + if err != nil { + return err + } + } + + return nil +} + +func run(plugins []plugin.Plugin) error { + for _, pl := range plugins { + err := runIPLDPlugin(pl) + if err != nil { + return err + } + } + return nil +} + +func runIPLDPlugin(pl plugin.Plugin) error { + ipldpl, ok := pl.(plugin.PluginIPLD) + if !ok { + return nil + } + + err := ipldpl.RegisterBlockDecoders(format.DefaultBlockDecoder) + if err != nil { + return err + } + + return ipldpl.RegisterInputEncParsers(coredag.DefaultInputEncParsers) +} diff --git a/plugin/loader/load.go b/plugin/loader/load.go new file mode 100644 index 00000000000..8edd53675e6 --- /dev/null +++ b/plugin/loader/load.go @@ -0,0 +1,65 @@ +package loader + +import ( + "fmt" + "os" + + "github.com/ipfs/go-ipfs/plugin" + + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" +) + +var log = logging.Logger("plugin/loader") + +var loadPluginsFunc = func(string) ([]plugin.Plugin, error) { + return nil, nil +} + +// LoadPlugins loads and initializes plugins. +func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) { + plMap := make(map[string]plugin.Plugin) + for _, v := range preloadPlugins { + plMap[v.Name()] = v + } + + newPls, err := loadDynamicPlugins(pluginDir) + if err != nil { + return nil, err + } + + for _, pl := range newPls { + if ppl, ok := plMap[pl.Name()]; ok { + // plugin is already preloaded + return nil, fmt.Errorf( + "plugin: %s, is duplicated in version: %s, "+ + "while trying to load dynamically: %s", + ppl.Name(), ppl.Version(), pl.Version()) + } + plMap[pl.Name()] = pl + } + + pls := make([]plugin.Plugin, 0, len(plMap)) + for _, v := range plMap { + pls = append(pls, v) + } + + err = initialize(pls) + if err != nil { + return nil, err + } + + err = run(pls) + return nil, err +} + +func loadDynamicPlugins(pluginDir string) ([]plugin.Plugin, error) { + _, err := os.Stat(pluginDir) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + + return loadPluginsFunc(pluginDir) +} diff --git a/plugin/loader/load_linux.go b/plugin/loader/load_linux.go new file mode 100644 index 00000000000..4d1a18cf8de --- /dev/null +++ b/plugin/loader/load_linux.go @@ -0,0 +1,67 @@ +package loader + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "plugin" + + iplugin "github.com/ipfs/go-ipfs/plugin" +) + +func init() { + loadPluginsFunc = linuxLoadFunc +} + +func linuxLoadFunc(pluginDir string) ([]iplugin.Plugin, error) { + var plugins []iplugin.Plugin + + err := filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + if fi != pluginDir { + log.Warningf("found directory inside plugins directory: %s", fi) + } + return nil + } + + if info.Mode().Perm()&0111 == 0 { + // file is not executable let's not load it + // this is to prevent loading plugins from for example non-executable + // mounts, some /tmp mounts are marked as such for security + log.Warningf("non-executable file in plugins directory: %s", fi) + return nil + } + + if newPlugins, err := loadPlugin(fi); err == nil { + plugins = append(plugins, newPlugins...) + } else { + return fmt.Errorf("loading plugin %s: %s", fi, err) + } + return nil + }) + + return plugins, err +} + +func loadPlugin(fi string) ([]iplugin.Plugin, error) { + pl, err := plugin.Open(fi) + if err != nil { + return nil, err + } + pls, err := pl.Lookup("Plugins") + if err != nil { + return nil, err + } + log.Errorf("plugins: %T", pls) + + typePls, ok := pls.(*[]iplugin.Plugin) + if !ok { + return nil, errors.New("filed 'Plugins' didn't contain correct type") + } + + return *typePls, nil +} diff --git a/plugin/loader/preload.sh b/plugin/loader/preload.sh new file mode 100755 index 00000000000..93ee3e43bb8 --- /dev/null +++ b/plugin/loader/preload.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +to_preload() { + awk 'NF' "$DIR/preload_list" | sed '/^#/d' +} + +cat </dev/null + +$($(d)_plugins_so): %.so : %/main/main.go +$($(d)_plugins_so): $$(DEPS_GO) ALWAYS + go build -buildmode=plugin -i -pkgdir "$(GOPATH)/pkg/linux_amd64_dynlink" $(go-flags-with-tags) -o "$@" "$(call go-pkg-name,$(basename $@))/main" + chmod +x "$@" + +CLEAN += $($(d)_plugins_so) + +build_plugins: $($(d)_plugins_so) + + +include mk/footer.mk diff --git a/plugin/plugins/gen_main.sh b/plugin/plugins/gen_main.sh new file mode 100755 index 00000000000..6ea705cfe9b --- /dev/null +++ b/plugin/plugins/gen_main.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +dir=${1:?first paramater with dir to work in is required} +pkg=${2:?second parameter with full name of the package is required} +main_pkg="$dir/main" + +shortpkg="uniquepkgname" + +mkdir -p "$main_pkg" + +cat > "$main_pkg/main.go" </dev/null 2>&1 && test_set_prereq DOCKER - -TEST_OS=$(uname -s | tr [a-z] [A-Z]) +test "$TEST_NO_PLUGIN" != 1 && test "$TEST_OS" = "LINUX" && test_set_prereq PLUGIN # Set a prereq as error messages are often different on Windows/Cygwin expr "$TEST_OS" : "CYGWIN_NT" >/dev/null || test_set_prereq STD_ERR_MSG @@ -53,6 +54,7 @@ expr "$TEST_OS" : "CYGWIN_NT" >/dev/null || test_set_prereq STD_ERR_MSG if test "$TEST_VERBOSE" = 1; then echo '# TEST_VERBOSE='"$TEST_VERBOSE" echo '# TEST_NO_FUSE='"$TEST_NO_FUSE" + echo '# TEST_NO_PLUGIN='"$TEST_NO_PLUGIN" echo '# TEST_EXPENSIVE='"$TEST_EXPENSIVE" echo '# TEST_OS='"$TEST_OS" fi diff --git a/test/sharness/t0280-plugin-git-data/git.tar.gz b/test/sharness/t0280-plugin-git-data/git.tar.gz new file mode 100644 index 00000000000..ff5298b7785 Binary files /dev/null and b/test/sharness/t0280-plugin-git-data/git.tar.gz differ diff --git a/test/sharness/t0280-plugin-git.sh b/test/sharness/t0280-plugin-git.sh new file mode 100755 index 00000000000..f09efed66d4 --- /dev/null +++ b/test/sharness/t0280-plugin-git.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# +# Copyright (c) 2017 Jakub Sztandera +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test git plugin" + +. lib/test-lib.sh + +# if in travis CI, dont test mount (no fuse) +if ! test_have_prereq PLUGIN; then + skip_all='skipping git plugin tests, plugins not available' + + test_done +fi + +test_init_ipfs + +test_expect_success "copy plugin" ' + mkdir -p "$IPFS_PATH/plugins" && + cp ../plugins/git.so "$IPFS_PATH/plugins/" +' + +# from https://github.com/ipfs/go-ipld-git/blob/master/make-test-repo.sh +test_expect_success "prepare test data" ' + tar xzf ../t0280-plugin-git-data/git.tar.gz +' + +test_dag_git() { + test_expect_success "add objects via dag put" ' + find objects -type f -exec ipfs dag put --format=git --input-enc=zlib {} \; -exec echo \; > hashes + ' + + test_expect_success "successfully get added objects" ' + cat hashes | xargs -i ipfs dag get -- {} > /dev/null + ' + + test_expect_success "path traversals work" ' + echo \"YmxvYiA3ACcsLnB5Zgo=\" > file1 && + ipfs dag get z8mWaJh5RLq16Zwgtd8gZxd63P4hgwNNx/object/parents/0/tree/dir2/hash/f3/hash > out1 + ' + + test_expect_success "outputs look correct" ' + test_cmp file1 out1 + ' +} + +# should work offline +#test_dag_git + +# should work online +test_launch_ipfs_daemon +test_dag_git +test_kill_ipfs_daemon + +test_done