NOISSUE - Add Things Migration

.env

@@ -0,0 +1,43 @@
+# Docker: Environment variables in Compose
+
+## Core Services
+MF_LOG_LEVEL=debug
+
+### Users
+MF_USERS_DB_HOST=localhost
+MF_USERS_DB_PORT=5432
+MF_USERS_DB_USER=mainflux
+MF_USERS_DB_PASS=mainflux
+MF_USERS_DB=users
+MF_USERS_DB_SSL_MODE=
+MF_USERS_DB_SSL_CERT=
+MF_USERS_DB_SSL_KEY=
+MF_USERS_DB_SSL_ROOT_CERT=
+MF_USERS_CONTAINER_NAME=mainflux-users-db
+MF_USERS_CONTAINER_NETWORK=docker_mainflux-base-net
+MF_USERS_CSV_PATH=csv/users.csv
+
+### Things
+MF_THINGS_DB_HOST=localhost
+MF_THINGS_DB_PORT=5432
+MF_THINGS_DB_USER=mainflux
+MF_THINGS_DB_PASS=mainflux
+MF_THINGS_DB=things
+MF_THINGS_DB_SSL_MODE=
+MF_THINGS_DB_SSL_CERT=
+MF_THINGS_DB_SSL_KEY=
+MF_THINGS_DB_SSL_ROOT_CERT=
+MF_THINGS_CONTAINER_NAME=mainflux-things-db
+MF_THINGS_CONTAINER_NETWORK=docker_mainflux-base-net
+MF_THINGS_CSV_PATH=csv/things.csv
+MF_CHANNELS_CSV_PATH=csv/channels.csv
+MF_CONNECTIONS_CSV_PATH=csv/connections.csv
+
+### SDK
+MF_USERS_URL=http://localhost
+MF_THINGS_URL=http://localhost
+[email protected]
+MF_USER_SECRET=12345678
+
+# Docker image tag
+MF_RELEASE_TAG=latest

.gitignore

@@ -10,3 +10,12 @@
 *.out
 # Go workspace file
 go.work
+
+# Binaries
+/build
+
+# CSV Output
+/csv
+
+# JSON benchmark outputs
+/scripts/*.json

Makefile

@@ -0,0 +1,35 @@
+BUILD_DIR = build
+SERVICES = migrate
+CGO_ENABLED ?= 0
+GOARCH ?= amd64
+COMMIT ?= $(shell git rev-parse HEAD)
+TIME ?= $(shell date +%F_%T)
+
+define compile_service
+	CGO_ENABLED=$(CGO_ENABLED) GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
+	go build -mod=vendor -ldflags "-s -w \
+	-X 'github.com/mainflux/mainflux.BuildTime=$(TIME)' \
+	-X 'github.com/mainflux/mainflux.Commit=$(COMMIT)'" \
+	-o ${BUILD_DIR}/mainflux-$(1) cmd/main.go
+endef
+
+all: $(SERVICES)
+
+.PHONY: all $(SERVICES)
+
+clean:
+	rm -rf ${BUILD_DIR}
+
+install:
+	cp ${BUILD_DIR}/* $(GOBIN)
+
+test:
+	go test -mod=vendor -v -race -count 1 -tags test $(shell go list ./... | grep -v 'vendor\|cmd')
+
+
+$(SERVICES):
+	$(call compile_service,$(@))
+
+
+changelog:
+	git log $(shell git describe --tags --abbrev=0)..HEAD --pretty=format:"- %s"

README.md

@@ -1,9 +1,137 @@
-# Migrations
+# Mainflux Migration Tool
 
-Scripts and tools used to migrate from one version of [mainflux][mainfluxLink] to another.
+A tool that is used to migrate from one version of [mainflux][mainflux-url] to another.
+
+## Installation
+
+```bash
+git clone https://github.com:mainflux/migrations.git
+cd migrations
+make migrate
+```
 
 ## Usage
 
-## Examples
+```bash
+Tool for migrating from one version of mainflux to another.It migrates things, channels and thier connections.
+                                Complete documentation is available at https://docs.mainflux.io
+
+Usage:
+  migrations [flags]
+
+Flags:
+  -f, --fromversion string   mainflux version you want to migrate from (default "0.13.0")
+  -h, --help                 help for migrations
+  -o, --operation string     export data from an existing mainflux deployment or import data to a new mainflux deployment (default "export")
+  -t, --toversion string     mainflux version you want to migrate to (default "0.14.0")
+```
+
+## Example
+
+### 1. Export From Version 0.13.0
+
+Make sure you have started mainflux deployment with version 0.13.0
+
+```bash
+./build/mainflux-migrate -f 0.13.0 -o export
+
+{"level":"info","message":"starting export from version 0.13.0","ts":"2023-03-30T14:32:53.725849074Z"}
+{"level":"debug","message":"connected to users database","ts":"2023-03-30T14:32:53.730192129Z"}
+{"level":"debug","message":"connected to things database","ts":"2023-03-30T14:32:53.737027714Z"}
+
+ ✓  Finished Retrieveing Users
+ ✓  Finished Retrieveing Things
+ ✓  Finished Retrieveing Channels
+ ✓  Finished Retrieveing Connection
+{"level":"info","message":"finished exporting from version 0.13.0","ts":"2023-03-30T14:32:55.816558105Z"}
+```
+
+### 2. Import To Version 0.14.0
+
+Make sure you have started mainflux deployment with version 0.14.0
+
+```bash
+./build/mainflux-migrate -t 0.14.0 -o import
+
+{"level":"info","message":"starting importing to version 0.14.0","ts":"2023-03-21T15:24:09.439589085Z"}
+{"level":"debug","message":"created user token","ts":"2023-03-21T15:24:09.508898562Z"}
+{"level":"debug","message":"created things","ts":"2023-03-21T15:24:12.748550379Z"}
+{"level":"debug","message":"created channels","ts":"2023-03-21T15:24:15.504673694Z"}
+{"level":"debug","message":"created connections","ts":"2023-03-21T15:26:41.718172029Z"}
+{"level":"info","message":"finished importing to version 0.14.0","ts":"2023-03-21T15:26:41.719890531Z"} 
+```
+
+## Testing
+
+If you want to seed the database from version 0.13.0 you can run
+
+```bash
+max=10
+for i in $(bash -c "echo {1..${max}}"); do ./provision -u [email protected] -p 12345678 --num 50 --prefix seed; done
+```
+
+This will create things and channels and connect them
+
+## Benchmarks
+
+Tools used to benchmark are:
+
+- [hyperfine][hyperfine-url]
+- [docker-compose][docker-compose-url]
+
+To run the benchmark tool run:
+
+```bash
+bash scripts/benchmark.sh "./build/mainflux-migrate -f 0.13.0 -o export"
+```
+
+The above example is to run migration benchmark during exporting
+
+Some of the benchmark results are:
+
+### Exporting
+
+| No. of Users | No. of Things | No. of Channels | No. of Connections | Avg Time(s)          | Max Time(s)          | Description                                 |
+| ------------ | ------------- | --------------- | ------------------ | -------------------- | -------------------- | ------------------------------------------- |
+| 1            | 10            | 10              | 100                | 0.026298354760000003 | 0.029383215860000002 | Each user has 10 things and 10 channels     |
+| 1            | 100           | 100             | 10K                | 0.05340002348        | 0.06367090818        | Each user has 100 things and 100 channels   |
+| 1            | 1000          | 1000            | 1M                 | 0.05664883321999999  | 0.06368840492        | Each user has 1000 things and 1000 channels |
+| 10           | 100           | 100             | 10K                | 0.03275979294        | 0.03625997544        | Each user has 10 things and 10 channels     |
+| 10           | 1K            | 1K              | 1M                 | 0.3047701865         | 0.34431983000000005  | Each user has 100 things and 100 channels   |
+| 10           | 10K           | 10K             | 100M               | 0.27441669116        | 0.29463570096        | Each user has 1000 things and 1000 channels |
+| 100          | 1K            | 1K              | 1M                 | 0.05666465024        | 0.06114762994        | Each user has 10 things and 10 channels     |
+| 100          | 10K           | 10K             | 100M               | 2.43419548986        | 2.90670491576        | Each user has 100 things and 100 channels   |
+
+Example of log output is:
+
+```bash
+Stopping Docker Compose...
+Starting Docker Compose...
+Provisioning 1 users, 10 things and 10 channels on Mainflux...
+Stopping Docker Compose...
+Starting Docker Compose...
+Provisioning 1 users, 100 things and 100 channels on Mainflux...
+Stopping Docker Compose...
+Starting Docker Compose...
+Provisioning 1 users, 1000 things and 1000 channels on Mainflux...
+Stopping Docker Compose...
+Starting Docker Compose...
+Provisioning 10 users, 10 things and 10 channels on Mainflux...
+Stopping Docker Compose...
+Starting Docker Compose...
+Provisioning 10 users, 100 things and 100 channels on Mainflux...
+Stopping Docker Compose...
+Starting Docker Compose...
+Provisioning 10 users, 1000 things and 1000 channels on Mainflux...
+Stopping Docker Compose...
+Starting Docker Compose...
+Provisioning 100 users, 10 things and 10 channels on Mainflux...
+Stopping Docker Compose...
+Starting Docker Compose...
+Provisioning 100 users, 100 things and 100 channels on Mainflux...
+Stopping Docker Compose...
+```
 
-[mainfluxLink]: https://github.com/mainflux/mainflux
+[mainflux-url]: https://github.com/mainflux/mainflux
+[hyperfine-url]: https://github.com/sharkdp/hyperfine/
+[docker-compose-url]: https://docs.docker.com/compose/

cmd/main.go

@@ -1 +1,48 @@
-package cmd
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	mflog "github.com/mainflux/mainflux/logger"
+	"github.com/mainflux/migrations/migrate"
+	"github.com/spf13/cobra"
+)
+
+const (
+	migrationDuration = 1 * time.Hour
+)
+
+func main() {
+	cfg := migrate.LoadConfig()
+
+	logger, err := mflog.New(os.Stdout, cfg.LogLevel)
+	if err != nil {
+		log.Fatalf(fmt.Sprintf("failed to init logger: %s", err.Error()))
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), migrationDuration)
+	defer cancel()
+
+	var rootCmd = &cobra.Command{
+		Use:   "migrations",
+		Short: "migrations is migration tool for Mainflux",
+		Long: `Tool for migrating from one version of mainflux to another.It migrates things, channels and their connections.
+				Complete documentation is available at https://docs.mainflux.io`,
+		Run: func(_ *cobra.Command, _ []string) {
+			migrate.Migrate(ctx, cfg, logger)
+		},
+	}
+
+	// Root Flags
+	rootCmd.PersistentFlags().StringVarP(&cfg.FromVersion, "fromversion", "f", "0.13.0", "mainflux version you want to migrate from")
+	rootCmd.PersistentFlags().StringVarP(&cfg.ToVersion, "toversion", "t", "0.14.0", "mainflux version you want to migrate to")
+	rootCmd.PersistentFlags().StringVarP(&cfg.Operation, "operation", "o", "export", "export data from an existing mainflux deployment or import data to a new mainflux deployment")
+
+	if err := rootCmd.Execute(); err != nil {
+		logger.Error(err.Error())
+	}
+}

config.go

@@ -0,0 +1,35 @@
+package migrations
+
+import (
+	thingspostgres "github.com/mainflux/mainflux/things/postgres" // for version 0.10.0, 0.11.0, 0.12.0 and 0.13.0
+	userspostgres "github.com/mainflux/mainflux/users/postgres"   // for version 0.10.0, 0.11.0, 0.12.0 and 0.13.0
+)
+
+type ThingsConfig struct {
+	DBConfig           thingspostgres.Config
+	ThingsCSVPath      string
+	ChannelsCSVPath    string
+	ConnectionsCSVPath string
+}
+
+type UsersConfig struct {
+	DBConfig     userspostgres.Config
+	UsersCSVPath string
+}
+
+type Config struct {
+	LogLevel                string
+	FromVersion             string
+	ToVersion               string
+	Operation               string
+	ThingsConfig            ThingsConfig
+	UsersConfig             UsersConfig
+	UsersURL                string
+	ThingsURL               string
+	UserIdentity            string
+	UserSecret              string
+	UsersRetrievalSQL       string
+	ThingsRetrievalSQL      string
+	ChannelsRetrievalSQL    string
+	ConnectionsRetrievalSQL string
+}

go.mod

@@ -1,3 +1,70 @@
 module github.com/mainflux/migrations
 
-go 1.20
+go 1.19
+
+require (
+	github.com/fatih/color v1.13.0
+	github.com/fsouza/go-dockerclient v1.9.6
+	github.com/gosuri/uilive v0.0.4
+	github.com/jmoiron/sqlx v1.3.4
+	github.com/julz/prettyprogress v0.0.0-20191213134954-46101f6567ae
+	github.com/mainflux/mainflux v0.0.0-20220415135135-92d8fb99bf82
+	github.com/mainflux/mainflux/pkg/sdk/go/0140 v0.0.0-00010101000000-000000000000
+	github.com/spf13/cobra v1.3.0
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
+)
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/containerd/containerd v1.6.18 // indirect
+	github.com/docker/docker v23.0.1+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/go-gorp/gorp/v3 v3.0.2 // indirect
+	github.com/go-kit/kit v0.12.0 // indirect
+	github.com/go-kit/log v0.2.0 // indirect
+	github.com/go-logfmt/logfmt v0.5.1 // indirect
+	github.com/gofrs/uuid v4.2.0+incompatible // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/klauspost/compress v1.14.2 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/lib/pq v1.10.4 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/moby/patternmatcher v0.5.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/oklog/ulid/v2 v2.0.2 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
+	github.com/opencontainers/runc v1.1.2 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/ory/keto/proto/ory/keto/acl/v1alpha1 v0.0.0-20210616104402-80e043246cf9 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rubenv/sql-migrate v1.1.1 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.1 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/net v0.6.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/tools v0.1.12 // indirect
+	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
+	google.golang.org/grpc v1.47.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+)
+
+replace (
+	github.com/mainflux/mainflux/0130 => github.com/mainflux/mainflux v0.0.0-20220415135135-92d8fb99bf82
+	github.com/mainflux/mainflux/pkg/sdk/go/0120 => github.com/rodneyosodo/mainflux/pkg/sdk/go v0.0.0-20230426122212-9899643bd719
+	github.com/mainflux/mainflux/pkg/sdk/go/0130 => github.com/rodneyosodo/mainflux/pkg/sdk/go v0.0.0-20230426122836-7ae32ac3815a
+	github.com/mainflux/mainflux/pkg/sdk/go/0140 => github.com/rodneyosodo/mainflux/pkg/sdk/go v0.0.0-20230317163628-6067ddd6a2fc
+	github.com/mainflux/mainflux/sdk/go/0100 => github.com/rodneyosodo/mainflux/sdk/go v0.0.0-20230426120131-f494ff385908
+	github.com/mainflux/mainflux/sdk/go/0110 => github.com/rodneyosodo/mainflux/sdk/go v0.0.0-20230426121329-d76305f507a5
+)