plugin: create plugin API and loader, add ipld-git plugin

License: MIT
Signed-off-by: Jeromy <[email protected]>

Rules.mk

@@ -56,6 +56,9 @@ include $(dir)/Rules.mk
 dir := pin/internal/pb
 include $(dir)/Rules.mk
 
+dir := plugin
+include $(dir)/Rules.mk
+
 # -------------------- #
 #   universal rules    #
 # -------------------- #

cmd/ipfs/main.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime/pprof"
 	"strings"
 	"sync"
@@ -22,6 +23,7 @@ import (
 	cmdsHttp "github.com/ipfs/go-ipfs/commands/http"
 	core "github.com/ipfs/go-ipfs/core"
 	coreCmds "github.com/ipfs/go-ipfs/core/commands"
+	"github.com/ipfs/go-ipfs/plugin/loader"
 	repo "github.com/ipfs/go-ipfs/repo"
 	config "github.com/ipfs/go-ipfs/repo/config"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
@@ -339,6 +341,11 @@ func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd
 	} else {
 		log.Debug("executing command locally")
 
+		pluginpath := filepath.Join(req.InvocContext().ConfigRoot, "plugins")
+		if _, err := loader.LoadPlugins(pluginpath); err != nil {
+			return nil, err
+		}
+
 		err := req.SetRootContext(ctx)
 		if err != nil {
 			return nil, err

core/commands/dag/dag.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
+	coredag "github.com/ipfs/go-ipfs/core/coredag"
 	path "github.com/ipfs/go-ipfs/path"
 	pin "github.com/ipfs/go-ipfs/pin"
 
@@ -76,34 +77,25 @@ into an object of the specified format.
 			defer n.Blockstore.PinLock().Unlock()
 		}
 
-		var c *cid.Cid
-		switch ienc {
-		case "json":
-			nd, err := convertJsonToType(fi, format)
-			if err != nil {
-				res.SetError(err, cmds.ErrNormal)
-				return
-			}
+		nds, err := coredag.ParseInputs(ienc, format, fi)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
 
-			c, err = n.DAG.Add(nd)
-			if err != nil {
-				res.SetError(err, cmds.ErrNormal)
-				return
-			}
-		case "raw":
-			nd, err := convertRawToType(fi, format)
+		var c *cid.Cid
+		b := n.DAG.Batch()
+		for _, nd := range nds {
+			cid, err := b.Add(nd)
 			if err != nil {
 				res.SetError(err, cmds.ErrNormal)
 				return
 			}
 
-			c, err = n.DAG.Add(nd)
-			if err != nil {
-				res.SetError(err, cmds.ErrNormal)
-				return
-			}
-		default:
-			res.SetError(fmt.Errorf("unrecognized input encoding: %s", ienc), cmds.ErrNormal)
+			c = cid
+		}
+		if err := b.Commit(); err != nil {
+			res.SetError(err, cmds.ErrNormal)
 			return
 		}
 

core/coredag/dagtransl.go

@@ -0,0 +1,81 @@
+package coredag
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	node "gx/ipfs/QmUBtPvHKFAX43XMsyxsYpMi3U5VwZ4jYFTo4kFhvAR33G/go-ipld-format"
+	ipldcbor "gx/ipfs/Qmcdid3XrCxcoNQUqZKiiKtM7JXxtyipU3izyRqwjFbVWw/go-ipld-cbor"
+)
+
+type DagParser func(r io.Reader) ([]node.Node, error)
+
+type FormatParsers map[string]DagParser
+type InputEncParsers map[string]FormatParsers
+
+var DefaultInputEncParsers = InputEncParsers{
+	"json": DefaultJsonParsers,
+	"raw":  DefaultRawParsers,
+}
+
+var DefaultJsonParsers = FormatParsers{
+	"cbor":     CborJsonParser,
+	"dag-cbor": CborJsonParser,
+}
+
+var DefaultRawParsers = FormatParsers{
+	"cbor":     CborRawParser,
+	"dag-cbor": CborRawParser,
+}
+
+func ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
+	return DefaultInputEncParsers.ParseInputs(ienc, format, r)
+}
+
+func (iep InputEncParsers) AddParser(ienv, format string, f DagParser) {
+	m, ok := iep[ienv]
+	if !ok {
+		m = make(FormatParsers)
+		iep[ienv] = m
+	}
+
+	m[format] = f
+}
+
+func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
+	pset, ok := iep[ienc]
+	if !ok {
+		return nil, fmt.Errorf("no input parser for %q", ienc)
+	}
+
+	parser, ok := pset[format]
+	if !ok {
+		return nil, fmt.Errorf("no parser for format %q using input type %q", format, ienc)
+	}
+
+	return parser(r)
+}
+
+func CborJsonParser(r io.Reader) ([]node.Node, error) {
+	nd, err := ipldcbor.FromJson(r)
+	if err != nil {
+		return nil, err
+	}
+
+	return []node.Node{nd}, nil
+}
+
+func CborRawParser(r io.Reader) ([]node.Node, error) {
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	nd, err := ipldcbor.Decode(data)
+	if err != nil {
+		return nil, err
+	}
+
+	return []node.Node{nd}, nil
+}

package.json

@@ -436,6 +436,12 @@
       "hash": "QmXxGS5QsUxpR3iqL5DjmsYPHR1Yz74siRQ4ChJqWFosMh",
       "name": "go-block-format",
       "version": "0.1.2"
+    },
+    {
+      "author": "whyrusleeping",
+      "hash": "QmdZKFV1ppRwGi5VQixitK5V2ihkDXyFckxaotPgXknHv4",
+      "name": "go-ipld-git",
+      "version": "0.0.0"
     }
   ],
   "gxVersion": "0.10.0",

plugin/Rules.mk

@@ -0,0 +1,7 @@
+include mk/header.mk
+
+dir := $(d)/plugins
+include $(dir)/Rules.mk
+
+
+include mk/footer.mk

plugin/ipld.go

@@ -0,0 +1,14 @@
+package plugin
+
+import (
+	"github.com/ipfs/go-ipfs/core/coredag"
+
+	node "gx/ipfs/QmUBtPvHKFAX43XMsyxsYpMi3U5VwZ4jYFTo4kFhvAR33G/go-ipld-format"
+)
+
+type PluginIPLD interface {
+	Plugin
+
+	RegisterBlockDecoders(dec node.BlockDecoder) error
+	RegisterInputEncParsers(iec coredag.InputEncParsers) error
+}

plugin/loader/initializer.go

@@ -0,0 +1,45 @@
+package loader
+
+import (
+	"github.com/ipfs/go-ipfs/core/coredag"
+	"github.com/ipfs/go-ipfs/plugin"
+
+	format "gx/ipfs/QmUBtPvHKFAX43XMsyxsYpMi3U5VwZ4jYFTo4kFhvAR33G/go-ipld-format"
+)
+
+func initalize(plugins []plugin.Plugin) error {
+	for _, p := range plugins {
+		err := p.Init()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func run(plugins []plugin.Plugin) error {
+	for _, pl := range plugins {
+		err := runIPLDPlugin(pl)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func runIPLDPlugin(pl plugin.Plugin) error {
+	ipldpl, ok := pl.(plugin.PluginIPLD)
+	if !ok {
+		return nil
+	}
+
+	var err error
+	err = ipldpl.RegisterBlockDecoders(format.DefaultBlockDecoder)
+	if err != nil {
+		return err
+	}
+
+	err = ipldpl.RegisterInputEncParsers(coredag.DefaultInputEncParsers)
+	return err
+}

plugin/loader/load.go

@@ -0,0 +1,28 @@
+package loader
+
+import (
+	"github.com/ipfs/go-ipfs/plugin"
+
+	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
+)
+
+var log = logging.Logger("plugin/loader")
+
+var loadPluginsFunc = func(string) ([]plugin.Plugin, error) {
+	return nil, nil
+}
+
+func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) {
+	pls, err := loadPluginsFunc(pluginDir)
+	if err != nil {
+		return nil, err
+	}
+
+	err = initalize(pls)
+	if err != nil {
+		return nil, err
+	}
+
+	err = run(pls)
+	return nil, err
+}

plugin/loader/load_linux.go

@@ -0,0 +1,64 @@
+package loader
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"plugin"
+
+	iplugin "github.com/ipfs/go-ipfs/plugin"
+)
+
+func init() {
+	loadPluginsFunc = linxuLoadFunc
+}
+
+func linxuLoadFunc(pluginDir string) ([]iplugin.Plugin, error) {
+	var plugins []iplugin.Plugin
+
+	filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			log.Warningf("found directory inside plugins directory: %s", fi)
+			return nil
+		}
+
+		if info.Mode().Perm()&0111 == 0 {
+			// file is not executable let's not load it
+			// this is to prevent loading plugins from for example non-executable
+			// mounts, some /tmp mounts are marked as such for security
+			log.Warningf("non-executable file in plugins directory: %s", fi)
+			return nil
+		}
+
+		if newPlugins, err := loadPlugin(fi); err == nil {
+			plugins = append(plugins, newPlugins...)
+		} else {
+			return fmt.Errorf("loading plugin %s: %s", fi, err)
+		}
+		return nil
+	})
+
+	return plugins, nil
+}
+
+func loadPlugin(fi string) ([]iplugin.Plugin, error) {
+	pl, err := plugin.Open(fi)
+	if err != nil {
+		return nil, err
+	}
+	pls, err := pl.Lookup("Plugins")
+	if err != nil {
+		return nil, err
+	}
+
+	typePls, ok := pls.([]iplugin.Plugin)
+	if !ok {
+		return nil, errors.New("filed 'Plugins' didn't contain correct type")
+	}
+
+	return typePls, nil
+}

plugin/plugin.go

@@ -0,0 +1,12 @@
+package plugin
+
+// Plugin is base interface for all kinds of go-ipfs plugins
+// It will be included in interfaces of different Plugins
+type Plugin interface {
+	// Name should return uniqe name of the plugin
+	Name() string
+	// Version returns current version of the plugin
+	Version() string
+	// Init is called once when the Plugin is being loaded
+	Init() error
+}

plugin/plugins/.gitignore

@@ -0,0 +1 @@
+*.so

plugin/plugins/Rules.mk

@@ -0,0 +1,12 @@
+include mk/header.mk
+
+$(d)_plugins:=$(d)/git
+$(d)_plugins_so:=$(addsuffix .so,$($(d)_plugins))
+
+$($(d)_plugins_so): $$(DEPS_GO) ALWAYS
+	go build -buildmode=plugin -i $(go-flags-with-tags) -o "$@" "$(call go-pkg-name,$(basename $@))"
+
+build_plugins: $($(d)_plugins_so)
+
+
+include mk/footer.mk