new repo created

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,2 +1,88 @@
-# skyhive.mongosync
-mongosync utility
+# SocialTalents.MongoSync
+MongoDb data sync tool, inspired by Liquibase.
+Supports inserts, updates, drop, delete, create index, and eval of any script.
+Works for Mongo 3.4+ or later.
+
+# Installation in dev environment (windows)
+
+1. Create a folder wihtin a project, e.g. MongoSync and place install_windows.bat in it.
+2. Run install_windows.bat
+3. Fresh release will be downloaded and extracted to MongoSync.suo folder (.suo so git will ignore it)
+4. Add install_windows.bat to git so others can easily install it (and install_linux.sh if you deploy to linux)
+
+# Collect some changes
+
+Execute following command to export whole collection:
+```
+MongoSync export --conn mongodb://localhost/database --collection Config 
+```
+
+Execute following command to export some query collection:
+```
+MongoSync export --conn mongodb://localhost/database --collection Config --query {myProperty:2}
+```
+
+It is getting more tricky when you want to use quotes. For windows, use single quotation. For linux, you have to escape them with \:
+```
+#windows
+MongoSync export --conn mongodb://localhost/database --collection Config --query {myProperty:'argument'}
+#linux
+MongoSync export --conn mongodb://localhost/database --collection Config --query {myProperty:\'argument\'}
+```
+
+
+MongoSync will generate file which you need to include within your deployment, e.g:
+`636517244.Config.Insert.json`
+
+3rd component define operation. You can rename file to use different insert mode (see mongoimport documentation):
+```
+636517244.Config.Upsert.json
+636517244.Config.Merge.json
+```
+
+## Delete objects
+Add file with Delete operation, put a search query into body:
+`636517244.Config.Delete.json`
+File content:
+```
+{}
+```
+
+## Drop colleciton
+```
+636517244.Config.Drop.json
+(File content ignored)
+````
+
+## CreateIndex
+Add file with an index definition in body:
+```
+636517244.Config.CreateIndex.js
+{ Type: 1, IndexKey: 1 }, { unique: true, partialFilterExpression: { IndexKey: { $exists: true } } }
+```
+
+## Eval any javascript
+```
+636517244.Config.eval.js
+printjson(db.getCollectionNames());
+```
+
+# Deployment
+
+Here is script for linux, but for windows logic is the same. 
+
+Navigate to folder with scripts and json files, install mongosync tool, run import.
+
+In case you need to install dotnet core follow first 2 steps from official documentation: https://www.microsoft.com/net/learn/get-started/linuxubuntu 
+
+```
+cd /MongoSync
+sh install_linux.sh
+sh mongosync.sh import --conn mongodb://user:password@127.0.0.1:29017/database
+```
+
+For vsts, we use "Run shell commands or a script on a remote machine using SSH" step as Agent phase so it executed only once.
+
+# Issues
+
+Known issue: windows script contains hardcoded path to mongo 3.6 folder, you need to fix it manually or add mongo to path

SocialTalents.MongoSync.Console.sln

@@ -0,0 +1,31 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 15
+VisualStudioVersion = 15.0.27130.2020
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SocialTalents.MongoSync.XUnit", "SocialTalents.MongoSync.XUnit\SocialTalents.MongoSync.XUnit.csproj", "{93C79473-28D9-4B12-BE94-4F74AE84419A}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SocialTalents.MongoSync.Console", "SocialTalents.MongoSync.Console\SocialTalents.MongoSync.Console.csproj", "{79FDF064-571C-4D6B-BF6A-31D8BE3AACE2}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{93C79473-28D9-4B12-BE94-4F74AE84419A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{93C79473-28D9-4B12-BE94-4F74AE84419A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{93C79473-28D9-4B12-BE94-4F74AE84419A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{93C79473-28D9-4B12-BE94-4F74AE84419A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{79FDF064-571C-4D6B-BF6A-31D8BE3AACE2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{79FDF064-571C-4D6B-BF6A-31D8BE3AACE2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{79FDF064-571C-4D6B-BF6A-31D8BE3AACE2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{79FDF064-571C-4D6B-BF6A-31D8BE3AACE2}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {9B30BEE7-CCDC-4E1A-9984-4D79765AC3CF}
+	EndGlobalSection
+EndGlobal

SocialTalents.MongoSync.Console.v3.ncrunchsolution

@@ -0,0 +1,6 @@
+<SolutionConfiguration>
+  <Settings>
+    <AllowParallelTestExecution>True</AllowParallelTestExecution>
+    <SolutionConfigured>True</SolutionConfigured>
+  </Settings>
+</SolutionConfiguration>

SocialTalents.MongoSync.Console/636516615.MongoSync3.eval.js

@@ -0,0 +1 @@
+printjson(db.getCollectionNames());

SocialTalents.MongoSync.Console/Model/Command.cs

@@ -0,0 +1,77 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace SocialTalents.MongoSync.Console.Model
+{
+    public class Command
+    {
+        public string Connection { get; set; }
+        public string AuthenticationDatabase { get; set; }
+        public string AuthenticationDatabaseToCommandLine()
+        {
+            if (string.IsNullOrEmpty(AuthenticationDatabase)) {
+                return string.Empty;
+            }
+            return $" --authenticationDatabase {AuthenticationDatabase}";
+        }
+        public CommandType CommandType { get; set; }
+
+        public virtual void Execute()
+        {
+            throw new NotImplementedException();
+        }
+
+        public virtual void Parse(string[] args)
+        {
+            var parsingRules = ParsingRules();
+            string lastSegment = null;
+            foreach(var parameter in args)
+            {
+                if (lastSegment == null)
+                {
+                    string key = parameter.ToLower();
+                    if (parsingRules.ContainsKey(key))
+                    {
+                        lastSegment = key;
+                    }
+                }
+                else
+                {
+                    try
+                    {
+                        parsingRules[lastSegment](this, parameter);
+                    }
+                    catch (Exception ex)
+                    {
+                        Program.Console($"Cannot parse {lastSegment} parameter: {ex.Message}");
+                        throw ex;
+                    }
+                    lastSegment = null;
+                }
+            }
+        }
+
+        public virtual void Validate()
+        {
+            throw new NotImplementedException("Need to be overriden");
+        }
+
+        protected virtual Dictionary<string, Action<Command, string>> ParsingRules()
+        {
+            var result = new Dictionary<string, Action<Command, string>>();
+            result.Add("--authenticationdatabase", (cmd, arg) => cmd.AuthenticationDatabase = arg);
+            result.Add("--uri", (cmd, arg) => cmd.Connection = arg);
+            result.Add("--conn", (cmd, arg) =>
+            {
+                // Backward compatibility parameter
+                if (cmd.Connection != null)
+                {
+                    throw new ArgumentException("Cannot use both --uri or --conn argument, please use --uri");
+                } 
+                cmd.Connection = arg;
+            });
+            return result;
+        }
+    }
+}

SocialTalents.MongoSync.Console/Model/CommandType.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace SocialTalents.MongoSync.Console.Model
+{
+    public enum CommandType
+    {
+        None = 0,
+        Import,
+        Export,
+        Help    
+    }
+}

SocialTalents.MongoSync.Console/Model/ConnectionString.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Text;
+using MongoDB.Driver;
+using MongoConnectionString = MongoDB.Driver.Core.Configuration.ConnectionString;
+
+namespace SocialTalents.MongoSync.Console.Model
+{
+    public class ConnectionString
+    {
+        private readonly MongoConnectionString _connectionString;
+
+        public ConnectionString(string connection)
+        {
+            try { _connectionString = new MongoConnectionString(connection); }
+            catch (MongoConfigurationException exception) { throw new ArgumentException(exception.Message); }
+
+            if(Database == null)
+                throw new ArgumentException("Connection string should have a database name in it, e.g. localhost/mydatabase");
+        }
+
+        public string UserName => _connectionString.Username;
+        public string Password => _connectionString.Password;
+
+        public string Database => _connectionString.DatabaseName;
+        public IEnumerable<string> Hosts => _connectionString.Hosts.Select(endPoint =>
+            endPoint is DnsEndPoint dnsEndPoint
+                ? $"{dnsEndPoint.Host}:{dnsEndPoint.Port}"
+                : $"{endPoint}"
+        );
+
+        public string ToCommandLine()
+        {
+            StringBuilder sb = new StringBuilder();
+            // mongo.exe requires database name as first parameter, in this way it is easier to fix parameters
+            sb.Append($"--db {Database}");
+
+            // https://docs.mongodb.com/manual/reference/program/mongo/#cmdoption-mongo-host
+            var hostOrReplica = string.IsNullOrEmpty(_connectionString.ReplicaSet)
+                ? Hosts.First()
+                : $"{_connectionString.ReplicaSet}/{string.Join(",", Hosts)}";
+            sb.Append($" --host {hostOrReplica}");
+
+            if (!string.IsNullOrEmpty(UserName))
+            {
+                sb.Append($" --username {UserName}");
+                if (!string.IsNullOrEmpty(Password))
+                {
+                    sb.Append($" --password {Password}");
+                }
+            }
+
+            if (_connectionString.Ssl.HasValue && _connectionString.Ssl.Value)
+            {
+                sb.Append(" --ssl");
+            }
+            return sb.ToString();
+        }
+    }
+}

SocialTalents.MongoSync.Console/Model/ExportCommand.cs

@@ -0,0 +1,52 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace SocialTalents.MongoSync.Console.Model
+{
+    public class ExportCommand : Command
+    {
+        public string CollectionName { get; set; }
+        public string SearchQueryForExport { get; set; } = "{}";
+        public string TimePrefix { get; set; } = buildPrefix();
+
+        private static string buildPrefix()
+        {
+            // assuming 1 export for same collection every 100 s is enough to sort imports later
+            long ticksPerSecond = 10000000;
+            return (DateTime.UtcNow.Ticks / ticksPerSecond / 100).ToString();
+        }
+
+        public override void Validate()
+        {
+            if (string.IsNullOrEmpty(Connection))
+            {
+                throw new ArgumentException("Connection parameter required for export command");
+            }
+            if (string.IsNullOrEmpty(CollectionName))
+            {
+                throw new ArgumentException("CollectionName parameter required for export command");
+            }
+        }
+
+        protected override Dictionary<string, Action<Command, string>> ParsingRules()
+        {
+            var r = base.ParsingRules();
+            r.Add("--collection", (c, a) => (c as ExportCommand).CollectionName = a);
+            r.Add("--query", (c, arg) => (c as ExportCommand).SearchQueryForExport = arg);
+            return r;
+        }
+
+        public static string COMMAND = "mongoexport";
+
+        public override void Execute()
+        {
+            ConnectionString c = new ConnectionString(Connection);
+            string argument = $"{c.ToCommandLine()}{AuthenticationDatabaseToCommandLine()} --collection {CollectionName} --query {SearchQueryForExport} --type json " +
+                // assuming no chance to generate more than 1 file per 10 seconds
+                $"--out {TimePrefix}.{CollectionName}.Insert.json";
+
+            Program.Exec(COMMAND, argument);
+        }
+    }
+}

SocialTalents.MongoSync.Console/Model/HelpCommand.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace SocialTalents.MongoSync.Console.Model
+{
+    public class HelpCommand : Command
+    {
+        public override void Execute()
+        {
+            Program.Console("Usage:");
+            Program.Console("SocialTalents.MongoSync.Console <command> --uri Connection [--file file] [--collection collection] [--query 'query'] [--authenticationDatabase admin]");
+            Program.Console("command        help    Display help");
+            Program.Console("               Import  process file(s) specified");
+            Program.Console("               Export  Export collection to file (using optional query)");
+            Program.Console("connection     mongodb Connection String (mongo+srv not supported)");
+            Program.Console("file           file or files to use, e.g. countries.json or *.json");
+            Program.Console("               File name format: [Order].[Collection].[ImportMode].json");
+            Program.Console("query          Query to use to export data, default is '{}'");
+            Program.Console("authenticationDatabase          Database to use for authentication");
+        }
+
+        public override void Parse(string[] args)
+        {
+            // no need to parse anything for help command
+        }
+
+        public override void Validate()
+        {
+
+        }
+    }
+}