Mentix service (cs3org#755)

.gitignore

@@ -11,6 +11,9 @@
 # Visual Studio Code
 .vscode
 
+# GoLand
+.idea
+
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
@@ -35,3 +38,4 @@ docs/tech-doc-hugo
 docs/themes/
 
 dist/
+bin/

CONTRIBUTORS.md

@@ -17,3 +17,4 @@
 - Mohitty <[email protected]>
 - Thomas Boerger <[email protected]>
 - Miroslav Bauer <[email protected]>
+- Daniel Mueller <[email protected]>

docs/content/en/docs/Config/HTTP/Services/Mentix/GOCDB/_index.md

@@ -0,0 +1,27 @@
+---
+title: "gocdb"
+linkTitle: "gocdb"
+weight: 10
+description: >
+    Configuration for the GOCDB connector of the Mentix service
+---
+
+{{% pageinfo %}}
+When using the [GOCDB](https://wiki.egi.eu/wiki/GOCDB/Documentation_Index) connector, at least its address has to be configured.
+{{% /pageinfo %}}
+
+{{% dir name="address" type="string" default="" %}}
+The address of the GOCDB instance; must be a valid URL (e.g., http://gocdb.uni-muenster.de). **Note:** The public API must be reachable under `<address>/gocdbpi/public`.
+{{< highlight toml >}}
+[http.services.mentix.gocdb]
+address = "http://gocdb.example.com"
+{{< /highlight >}}
+{{% /dir %}}
+
+{{% dir name="scope" type="string" default="SM" %}}
+The scope to use for filtering sites and services.
+{{< highlight toml >}}
+[http.services.mentix.gocdb]
+scope = "SM"
+{{< /highlight >}}
+{{% /dir %}}

docs/content/en/docs/Config/HTTP/Services/Mentix/PromFileSD/_index.md

@@ -0,0 +1,19 @@
+---
+title: "prom_filesd"
+linkTitle: "prom_filesd"
+weight: 10
+description: >
+    Configuration for the Prometheus File SD exporter of the Mentix service
+---
+
+{{% pageinfo %}}
+When using the Prometheus File SD exporter, the output filename has to be configured first.
+{{% /pageinfo %}}
+
+{{% dir name="output_file" type="string" default="" %}}
+The target filename of the generated Prometheus File SD scrape config.
+{{< highlight toml >}}
+[http.services.mentix.prom_filesd]
+output_file = "/var/shared/prometheus/sciencemesh.json"
+{{< /highlight >}}
+{{% /dir %}}

docs/content/en/docs/Config/HTTP/Services/Mentix/WebAPI/_index.md

@@ -0,0 +1,19 @@
+---
+title: "webapi"
+linkTitle: "webapi"
+weight: 10
+description: >
+    Configuration for the WebAPI of the Mentix service
+---
+
+{{% pageinfo %}}
+The WebAPI exporter supports multiple endpoints for exporting data. As there currently is only one such endpoint, the WebAPI settings should not be modified.
+{{% /pageinfo %}}
+
+{{% dir name="endpoint" type="string" default="/" %}}
+The endpoint where the mesh data can be queried.
+{{< highlight toml >}}
+[http.services.mentix.webapi]
+endpoint = "data"
+{{< /highlight >}}
+{{% /dir %}}

docs/content/en/docs/Config/HTTP/Services/Mentix/_index.md

@@ -0,0 +1,63 @@
+---
+title: "mentix"
+linkTitle: "mentix"
+weight: 10
+description: >
+    Configuration for the Mentix service
+---
+
+{{% pageinfo %}}
+Mentix (_**Me**sh E**nti**ty E**x**porter_) is a service to read mesh topology data from a source (e.g., a GOCDB instance) and export it to various targets like an HTTP endpoint or Prometheus.
+{{% /pageinfo %}}
+
+{{% dir name="prefix" type="string" default="mentix" %}}
+The relative root path of all exposed HTTP endpoints of Mentix.
+{{< highlight toml >}}
+[http.services.mentix]
+prefix = "/mentix"
+{{< /highlight >}}
+{{% /dir %}}
+
+{{% dir name="connector" type="string" default="gocdb" %}}
+Mentix is decoupled from the actual source of the mesh data by using a so-called _connector_. A connector is used to gather the data from a certain source, which are then converted into Mentix' own internal format.
+
+Supported values are:
+
+- **gocdb** 
+The [GOCDB](https://wiki.egi.eu/wiki/GOCDB/Documentation_Index) is a database specifically designed to organize the topology of a mesh of distributed sites and services. In order to use GOCDB with Mentix, its instance address has to be configured (see [here](gocdb)).    
+
+{{< highlight toml >}}
+[http.services.mentix]
+connector = "gocdb"
+{{< /highlight >}}
+{{% /dir %}}
+
+{{% dir name="exporters" type="[]string" default="[webapi,prom_filesd]" %}}
+Mentix exposes its gathered data by using one or more _exporters_. Such exporters can, for example, write the data to a file in a specific format, or offer the data via an HTTP endpoint.
+
+Supported values are:
+
+- **webapi**
+Mentix exposes its data via an HTTP endpoint using the `webapi` exporter. Data can be retrieved at the configured relative endpoint (see [here](webapi)). The web API currently doesn't support any parameters but will most likely be extended in the future.
+- **prom_filesd**
+[Prometheus](https://prometheus.io/) supports discovering new services it should monitor via external configuration files (hence, this is called _file-based service discovery_). Mentix can create such files using the `prom_filesd` exporter. To use this exporter, you have to specify the target output file in the configuration (see [here](prom_filesd)). You also have to set up the discovery service in Prometheus by adding a scrape configuration like the following example to the Prometheus configuration (for more information, visit the official [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config)):
+  ``` scrape_configs:
+      - job_name: 'sciencemesh'
+        file_sd_configs:
+          - files:
+             - '/usr/share/prom/sciencemesh_services.json'
+  ```
+
+{{< highlight toml >}}
+[http.services.mentix]
+exporters = ["webapi", "prom_filesd"]
+{{< /highlight >}}
+{{% /dir %}}
+
+{{% dir name="update_interval" type="string" default="1h" %}}
+How frequently Mentix should pull and update the mesh data. Supports common time duration strings, like "1h30m", "1d" etc.
+{{< highlight toml >}}
+[http.services.mentix]
+update_interval = "15m"
+{{< /highlight >}}
+{{% /dir %}}

examples/mentix/mentix.toml

@@ -0,0 +1,24 @@
+[shared]
+jwt_secret = "Ment1x-T0pS3cr3t"
+
+[http]
+address = "0.0.0.0:9600"
+enabled_services = ["mentix"]
+
+[http.services.mentix]
+connector = "gocdb"
+exporters = ["webapi"]
+# Enable the Prometheus File Service Discovery:
+# exporters = ["webapi", "prom_filesd"]
+update_interval = "15m"
+
+[http.services.mentix.gocdb]
+address = "http://sciencemesh-test.uni-muenster.de"
+
+[http.services.mentix.webapi]
+endpoint = "/"
+
+# Configure the Prometheus File Service Discovery:
+# [http.services.mentix.prom_filesd]
+# Prometheus must be configured to read the following file:
+# output_file = "/usr/share/prom/sciencemesh_services.json"

go.sum

@@ -137,6 +137,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=

internal/http/services/datagateway/datagateway.go

@@ -31,6 +31,7 @@ import (
 	"github.com/dgrijalva/jwt-go"
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 )
 
 const (
@@ -57,7 +58,7 @@ type svc struct {
 }
 
 // New returns a new datagateway
-func New(m map[string]interface{}) (global.Service, error) {
+func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) {
 	conf := &config{}
 	if err := mapstructure.Decode(m, conf); err != nil {
 		return nil, err

internal/http/services/dataprovider/dataprovider.go

@@ -27,6 +27,7 @@ import (
 	"github.com/cs3org/reva/pkg/storage"
 	"github.com/cs3org/reva/pkg/storage/fs/registry"
 	"github.com/mitchellh/mapstructure"
+	"github.com/rs/zerolog"
 	tusd "github.com/tus/tusd/pkg/handler"
 )
 
@@ -48,7 +49,7 @@ type svc struct {
 }
 
 // New returns a new datasvc
-func New(m map[string]interface{}) (global.Service, error) {
+func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) {
 	conf := &config{}
 	if err := mapstructure.Decode(m, conf); err != nil {
 		return nil, err

internal/http/services/helloworld/helloworld.go

@@ -24,14 +24,15 @@ import (
 	"github.com/cs3org/reva/pkg/appctx"
 	"github.com/cs3org/reva/pkg/rhttp/global"
 	"github.com/mitchellh/mapstructure"
+	"github.com/rs/zerolog"
 )
 
 func init() {
 	global.Register("helloworld", New)
 }
 
 // New returns a new helloworld service
-func New(m map[string]interface{}) (global.Service, error) {
+func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) {
 	conf := &config{}
 	if err := mapstructure.Decode(m, conf); err != nil {
 		return nil, err

internal/http/services/loader/loader.go

@@ -23,6 +23,7 @@ import (
 	_ "github.com/cs3org/reva/internal/http/services/datagateway"
 	_ "github.com/cs3org/reva/internal/http/services/dataprovider"
 	_ "github.com/cs3org/reva/internal/http/services/helloworld"
+	_ "github.com/cs3org/reva/internal/http/services/mentix"
 	_ "github.com/cs3org/reva/internal/http/services/meshdirectory"
 	_ "github.com/cs3org/reva/internal/http/services/ocmd"
 	_ "github.com/cs3org/reva/internal/http/services/oidcprovider"

internal/http/services/mentix/mentix.go

@@ -0,0 +1,146 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package mentix
+
+import (
+	"net/http"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+
+	"github.com/cs3org/reva/pkg/mentix"
+	"github.com/cs3org/reva/pkg/mentix/config"
+	"github.com/cs3org/reva/pkg/mentix/exporters"
+	"github.com/cs3org/reva/pkg/rhttp/global"
+)
+
+func init() {
+	global.Register(serviceName, New)
+}
+
+type svc struct {
+	conf *config.Configuration
+	mntx *mentix.Mentix
+	log  *zerolog.Logger
+
+	stopSignal chan struct{}
+}
+
+const (
+	serviceName = "mentix"
+)
+
+func (s *svc) Close() error {
+	// Trigger and close the stopSignal signal channel to stop Mentix
+	s.stopSignal <- struct{}{}
+	close(s.stopSignal)
+
+	return nil
+}
+
+func (s *svc) Prefix() string {
+	return s.conf.Prefix
+}
+
+func (s *svc) Unprotected() []string {
+	// Get all endpoints exposed by the RequestExporters
+	exporters := s.mntx.GetRequestExporters()
+	endpoints := make([]string, len(exporters))
+	for _, exporter := range exporters {
+		endpoints = append(endpoints, exporter.Endpoint())
+	}
+	return endpoints
+}
+
+func (s *svc) Handler() http.Handler {
+	// Forward requests to Mentix
+	return http.HandlerFunc(s.mntx.RequestHandler)
+}
+
+func (s *svc) startBackgroundService() {
+	// Just run Mentix in the background
+	go func() {
+		if err := s.mntx.Run(s.stopSignal); err != nil {
+			s.log.Err(err).Msg("error while running mentix")
+		}
+	}()
+}
+
+func parseConfig(m map[string]interface{}) (*config.Configuration, error) {
+	cfg := &config.Configuration{}
+	if err := mapstructure.Decode(m, &cfg); err != nil {
+		return nil, errors.Wrap(err, "mentix: error decoding configuration")
+	}
+	applyDefaultConfig(cfg)
+	return cfg, nil
+}
+
+func applyDefaultConfig(*config.Configuration) {
+	conf := &config.Configuration{}
+
+	if conf.Prefix == "" {
+		conf.Prefix = serviceName
+	}
+
+	if conf.Connector == "" {
+		conf.Connector = config.ConnectorIDGOCDB // Use GOCDB
+	}
+
+	if len(conf.Exporters) == 0 {
+		conf.Exporters = exporters.RegisteredExporterIDs() // Enable all exporters
+	}
+
+	if conf.UpdateInterval == "" {
+		conf.UpdateInterval = "1h" // Update once per hour
+	}
+
+	if conf.GOCDB.Scope == "" {
+		conf.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future
+	}
+
+	if conf.WebAPI.Endpoint == "" {
+		conf.WebAPI.Endpoint = "/"
+	}
+}
+
+// New returns a new Mentix service.
+func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) {
+	// Prepare the configuration
+	conf, err := parseConfig(m)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the Mentix instance
+	mntx, err := mentix.New(conf, log)
+	if err != nil {
+		return nil, errors.Wrap(err, "mentix: error creating Mentix")
+	}
+
+	// Create the service and start its background activity
+	s := &svc{
+		conf:       conf,
+		mntx:       mntx,
+		log:        log,
+		stopSignal: make(chan struct{}),
+	}
+	s.startBackgroundService()
+	return s, nil
+}