diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..354ef25 Binary files /dev/null and b/.DS_Store differ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ab60297 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 8bef9e6..7934434 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,88 @@ -# skyhive.mongosync -mongosync utility +# SocialTalents.MongoSync +MongoDb data sync tool, inspired by Liquibase. +Supports inserts, updates, drop, delete, create index, and eval of any script. +Works for Mongo 3.4+ or later. + +# Installation in dev environment (windows) + +1. Create a folder wihtin a project, e.g. MongoSync and place install_windows.bat in it. +2. Run install_windows.bat +3. Fresh release will be downloaded and extracted to MongoSync.suo folder (.suo so git will ignore it) +4. Add install_windows.bat to git so others can easily install it (and install_linux.sh if you deploy to linux) + +# Collect some changes + +Execute following command to export whole collection: +``` +MongoSync export --conn mongodb://localhost/database --collection Config +``` + +Execute following command to export some query collection: +``` +MongoSync export --conn mongodb://localhost/database --collection Config --query {myProperty:2} +``` + +It is getting more tricky when you want to use quotes. For windows, use single quotation. For linux, you have to escape them with \: +``` +#windows +MongoSync export --conn mongodb://localhost/database --collection Config --query {myProperty:'argument'} +#linux +MongoSync export --conn mongodb://localhost/database --collection Config --query {myProperty:\'argument\'} +``` + + +MongoSync will generate file which you need to include within your deployment, e.g: +`636517244.Config.Insert.json` + +3rd component define operation. You can rename file to use different insert mode (see mongoimport documentation): +``` +636517244.Config.Upsert.json +636517244.Config.Merge.json +``` + +## Delete objects +Add file with Delete operation, put a search query into body: +`636517244.Config.Delete.json` +File content: +``` +{} +``` + +## Drop colleciton +``` +636517244.Config.Drop.json +(File content ignored) +```` + +## CreateIndex +Add file with an index definition in body: +``` +636517244.Config.CreateIndex.js +{ Type: 1, IndexKey: 1 }, { unique: true, partialFilterExpression: { IndexKey: { $exists: true } } } +``` + +## Eval any javascript +``` +636517244.Config.eval.js +printjson(db.getCollectionNames()); +``` + +# Deployment + +Here is script for linux, but for windows logic is the same. + +Navigate to folder with scripts and json files, install mongosync tool, run import. + +In case you need to install dotnet core follow first 2 steps from official documentation: https://www.microsoft.com/net/learn/get-started/linuxubuntu + +``` +cd /MongoSync +sh install_linux.sh +sh mongosync.sh import --conn mongodb://user:password@127.0.0.1:29017/database +``` + +For vsts, we use "Run shell commands or a script on a remote machine using SSH" step as Agent phase so it executed only once. + +# Issues + +Known issue: windows script contains hardcoded path to mongo 3.6 folder, you need to fix it manually or add mongo to path diff --git a/SocialTalents.MongoSync.Console.sln b/SocialTalents.MongoSync.Console.sln new file mode 100644 index 0000000..0ddfdca --- /dev/null +++ b/SocialTalents.MongoSync.Console.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.27130.2020 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SocialTalents.MongoSync.XUnit", "SocialTalents.MongoSync.XUnit\SocialTalents.MongoSync.XUnit.csproj", "{93C79473-28D9-4B12-BE94-4F74AE84419A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SocialTalents.MongoSync.Console", "SocialTalents.MongoSync.Console\SocialTalents.MongoSync.Console.csproj", "{79FDF064-571C-4D6B-BF6A-31D8BE3AACE2}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {93C79473-28D9-4B12-BE94-4F74AE84419A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {93C79473-28D9-4B12-BE94-4F74AE84419A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {93C79473-28D9-4B12-BE94-4F74AE84419A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {93C79473-28D9-4B12-BE94-4F74AE84419A}.Release|Any CPU.Build.0 = Release|Any CPU + {79FDF064-571C-4D6B-BF6A-31D8BE3AACE2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {79FDF064-571C-4D6B-BF6A-31D8BE3AACE2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {79FDF064-571C-4D6B-BF6A-31D8BE3AACE2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {79FDF064-571C-4D6B-BF6A-31D8BE3AACE2}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {9B30BEE7-CCDC-4E1A-9984-4D79765AC3CF} + EndGlobalSection +EndGlobal diff --git a/SocialTalents.MongoSync.Console.v3.ncrunchsolution b/SocialTalents.MongoSync.Console.v3.ncrunchsolution new file mode 100644 index 0000000..10420ac --- /dev/null +++ b/SocialTalents.MongoSync.Console.v3.ncrunchsolution @@ -0,0 +1,6 @@ + + + True + True + + \ No newline at end of file diff --git a/SocialTalents.MongoSync.Console/.DS_Store b/SocialTalents.MongoSync.Console/.DS_Store new file mode 100644 index 0000000..224a349 Binary files /dev/null and b/SocialTalents.MongoSync.Console/.DS_Store differ diff --git a/SocialTalents.MongoSync.Console/636516615.MongoSync3.eval.js b/SocialTalents.MongoSync.Console/636516615.MongoSync3.eval.js new file mode 100644 index 0000000..027cdc3 --- /dev/null +++ b/SocialTalents.MongoSync.Console/636516615.MongoSync3.eval.js @@ -0,0 +1 @@ +printjson(db.getCollectionNames()); \ No newline at end of file diff --git a/SocialTalents.MongoSync.Console/Model/Command.cs b/SocialTalents.MongoSync.Console/Model/Command.cs new file mode 100644 index 0000000..35d1332 --- /dev/null +++ b/SocialTalents.MongoSync.Console/Model/Command.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace SocialTalents.MongoSync.Console.Model +{ + public class Command + { + public string Connection { get; set; } + public string AuthenticationDatabase { get; set; } + public string AuthenticationDatabaseToCommandLine() + { + if (string.IsNullOrEmpty(AuthenticationDatabase)) { + return string.Empty; + } + return $" --authenticationDatabase {AuthenticationDatabase}"; + } + public CommandType CommandType { get; set; } + + public virtual void Execute() + { + throw new NotImplementedException(); + } + + public virtual void Parse(string[] args) + { + var parsingRules = ParsingRules(); + string lastSegment = null; + foreach(var parameter in args) + { + if (lastSegment == null) + { + string key = parameter.ToLower(); + if (parsingRules.ContainsKey(key)) + { + lastSegment = key; + } + } + else + { + try + { + parsingRules[lastSegment](this, parameter); + } + catch (Exception ex) + { + Program.Console($"Cannot parse {lastSegment} parameter: {ex.Message}"); + throw ex; + } + lastSegment = null; + } + } + } + + public virtual void Validate() + { + throw new NotImplementedException("Need to be overriden"); + } + + protected virtual Dictionary> ParsingRules() + { + var result = new Dictionary>(); + result.Add("--authenticationdatabase", (cmd, arg) => cmd.AuthenticationDatabase = arg); + result.Add("--uri", (cmd, arg) => cmd.Connection = arg); + result.Add("--conn", (cmd, arg) => + { + // Backward compatibility parameter + if (cmd.Connection != null) + { + throw new ArgumentException("Cannot use both --uri or --conn argument, please use --uri"); + } + cmd.Connection = arg; + }); + return result; + } + } +} diff --git a/SocialTalents.MongoSync.Console/Model/CommandType.cs b/SocialTalents.MongoSync.Console/Model/CommandType.cs new file mode 100644 index 0000000..6912d7e --- /dev/null +++ b/SocialTalents.MongoSync.Console/Model/CommandType.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace SocialTalents.MongoSync.Console.Model +{ + public enum CommandType + { + None = 0, + Import, + Export, + Help + } +} diff --git a/SocialTalents.MongoSync.Console/Model/ConnectionString.cs b/SocialTalents.MongoSync.Console/Model/ConnectionString.cs new file mode 100644 index 0000000..10ea7be --- /dev/null +++ b/SocialTalents.MongoSync.Console/Model/ConnectionString.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text; +using MongoDB.Driver; +using MongoConnectionString = MongoDB.Driver.Core.Configuration.ConnectionString; + +namespace SocialTalents.MongoSync.Console.Model +{ + public class ConnectionString + { + private readonly MongoConnectionString _connectionString; + + public ConnectionString(string connection) + { + try { _connectionString = new MongoConnectionString(connection); } + catch (MongoConfigurationException exception) { throw new ArgumentException(exception.Message); } + + if(Database == null) + throw new ArgumentException("Connection string should have a database name in it, e.g. localhost/mydatabase"); + } + + public string UserName => _connectionString.Username; + public string Password => _connectionString.Password; + + public string Database => _connectionString.DatabaseName; + public IEnumerable Hosts => _connectionString.Hosts.Select(endPoint => + endPoint is DnsEndPoint dnsEndPoint + ? $"{dnsEndPoint.Host}:{dnsEndPoint.Port}" + : $"{endPoint}" + ); + + public string ToCommandLine() + { + StringBuilder sb = new StringBuilder(); + // mongo.exe requires database name as first parameter, in this way it is easier to fix parameters + sb.Append($"--db {Database}"); + + // https://docs.mongodb.com/manual/reference/program/mongo/#cmdoption-mongo-host + var hostOrReplica = string.IsNullOrEmpty(_connectionString.ReplicaSet) + ? Hosts.First() + : $"{_connectionString.ReplicaSet}/{string.Join(",", Hosts)}"; + sb.Append($" --host {hostOrReplica}"); + + if (!string.IsNullOrEmpty(UserName)) + { + sb.Append($" --username {UserName}"); + if (!string.IsNullOrEmpty(Password)) + { + sb.Append($" --password {Password}"); + } + } + + if (_connectionString.Ssl.HasValue && _connectionString.Ssl.Value) + { + sb.Append(" --ssl"); + } + return sb.ToString(); + } + } +} diff --git a/SocialTalents.MongoSync.Console/Model/ExportCommand.cs b/SocialTalents.MongoSync.Console/Model/ExportCommand.cs new file mode 100644 index 0000000..037e60f --- /dev/null +++ b/SocialTalents.MongoSync.Console/Model/ExportCommand.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace SocialTalents.MongoSync.Console.Model +{ + public class ExportCommand : Command + { + public string CollectionName { get; set; } + public string SearchQueryForExport { get; set; } = "{}"; + public string TimePrefix { get; set; } = buildPrefix(); + + private static string buildPrefix() + { + // assuming 1 export for same collection every 100 s is enough to sort imports later + long ticksPerSecond = 10000000; + return (DateTime.UtcNow.Ticks / ticksPerSecond / 100).ToString(); + } + + public override void Validate() + { + if (string.IsNullOrEmpty(Connection)) + { + throw new ArgumentException("Connection parameter required for export command"); + } + if (string.IsNullOrEmpty(CollectionName)) + { + throw new ArgumentException("CollectionName parameter required for export command"); + } + } + + protected override Dictionary> ParsingRules() + { + var r = base.ParsingRules(); + r.Add("--collection", (c, a) => (c as ExportCommand).CollectionName = a); + r.Add("--query", (c, arg) => (c as ExportCommand).SearchQueryForExport = arg); + return r; + } + + public static string COMMAND = "mongoexport"; + + public override void Execute() + { + ConnectionString c = new ConnectionString(Connection); + string argument = $"{c.ToCommandLine()}{AuthenticationDatabaseToCommandLine()} --collection {CollectionName} --query {SearchQueryForExport} --type json " + + // assuming no chance to generate more than 1 file per 10 seconds + $"--out {TimePrefix}.{CollectionName}.Insert.json"; + + Program.Exec(COMMAND, argument); + } + } +} diff --git a/SocialTalents.MongoSync.Console/Model/HelpCommand.cs b/SocialTalents.MongoSync.Console/Model/HelpCommand.cs new file mode 100644 index 0000000..89a6d7a --- /dev/null +++ b/SocialTalents.MongoSync.Console/Model/HelpCommand.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace SocialTalents.MongoSync.Console.Model +{ + public class HelpCommand : Command + { + public override void Execute() + { + Program.Console("Usage:"); + Program.Console("SocialTalents.MongoSync.Console --uri Connection [--file file] [--collection collection] [--query 'query'] [--authenticationDatabase admin]"); + Program.Console("command help Display help"); + Program.Console(" Import process file(s) specified"); + Program.Console(" Export Export collection to file (using optional query)"); + Program.Console("connection mongodb Connection String (mongo+srv not supported)"); + Program.Console("file file or files to use, e.g. countries.json or *.json"); + Program.Console(" File name format: [Order].[Collection].[ImportMode].json"); + Program.Console("query Query to use to export data, default is '{}'"); + Program.Console("authenticationDatabase Database to use for authentication"); + } + + public override void Parse(string[] args) + { + // no need to parse anything for help command + } + + public override void Validate() + { + + } + } +} diff --git a/SocialTalents.MongoSync.Console/Model/ImportCommand.cs b/SocialTalents.MongoSync.Console/Model/ImportCommand.cs new file mode 100644 index 0000000..053c9fb --- /dev/null +++ b/SocialTalents.MongoSync.Console/Model/ImportCommand.cs @@ -0,0 +1,205 @@ +using MongoDB.Bson; +using MongoDB.Driver; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace SocialTalents.MongoSync.Console.Model +{ + public class ImportCommand : Command + { + public ImportCommand() + { + // to cover .js and .json + FilesFilter = "*.js*"; + ReadCompletedImports = ReadCompletedImportsDefault; + ConnectToMongoDb = ConnectToMongoDbDefault; + } + + public ConnectionString ConnectionString { get; set; } + public string FilesFilter { get; set; } + + public override void Execute() + { + try + { + Dictionary completedImports = ReadCompletedImports(); + + var allFiles = ReadFiles(FilesFilter); + Program.Console($"Found {allFiles.Length} files to import"); + foreach (var f in allFiles.OrderBy(f => f.Name)) + { + if (completedImports.ContainsKey(f.Name)) + { + Program.Console($"Skipping {f.Name}, already imported on {completedImports[f.Name].Imported.ToString("u")}"); + continue; + } + + var fileNameSplit = f.Name.Split('.'); + if (fileNameSplit.Length != 4) + { + throw new InvalidOperationException($"File {f.Name} do not have 4 components: [Order].[Collection].[Operation].json"); + } + string collectionName = fileNameSplit[1]; + ImportMode importMode = Enum.Parse(fileNameSplit[2], true); + + Program.Console($"Importing {f.Name} to collection {collectionName}, mode {importMode}"); + + + SyncEntity importResult = new SyncEntity(); + importResult.FileName = f.Name; + + try + { + switch (importMode) + { + case ImportMode.Drop: + dropCollection(collectionName); + break; + case ImportMode.Delete: + deleteFromCollection(collectionName, f); + break; + case ImportMode.Insert: + case ImportMode.Upsert: + case ImportMode.Merge: + var resultCode = Program.Exec(IMPORT_COMMAND, $"{ConnectionString.ToCommandLine()}{AuthenticationDatabaseToCommandLine()} --collection {collectionName} --type json --mode {importMode.ToString().ToLower()} --stopOnError --file {f.Name}"); + if (resultCode != 0) + { + throw new InvalidOperationException($"mongoimport result code {resultCode}, interrupting"); + } + break; + case ImportMode.Eval: + var evalResultCode = Program.Exec(MONGO_COMMAND, $"{Connection} {f.Name}"); + if (evalResultCode != 0) + { + throw new InvalidOperationException($"mongo result code {evalResultCode}, interrupting"); + } + break; + case ImportMode.CreateIndex: + var text = File.ReadAllText(f.FullName); + + Program.Console("Index params:"); + Program.Console(text); + + var command = string.Format(CREATE_INDEX, collectionName, text); + var fileName = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".js"); + File.WriteAllText(fileName, command); + + try + { + var createIndexResultCode = Program.Exec(MONGO_COMMAND, $"{Connection} {fileName}"); + switch (createIndexResultCode) + { + // no error + case 0: + break; + case 11000: + throw new InvalidOperationException($"CreateIndex failed with error 'duplicate key error', interrupting"); + default: + // all error codes with explanation + // https://github.com/mongodb/mongo/blob/master/src/mongo/base/error_codes.err + throw new InvalidOperationException($"CreateIndex result code {createIndexResultCode}, interrupting"); + } + } + finally + { + File.Delete(fileName); + } + break; + default: throw new InvalidOperationException($"Import mode {importMode} not implemented yet"); + } + importResult.Success = true; + } + finally + { + SyncCollection.InsertOne(importResult); + } + } + Program.Console($"Import completed successfully"); + } + catch (Exception ex) + { + Program.Console($"Error during import: {ex.Message}"); + Environment.Exit(1); + } + } + + private void deleteFromCollection(string collectionName, FileInfo f) + { + var deleteCommand = System.IO.File.ReadAllLines(f.FullName); + foreach(string delete in deleteCommand) + { + if (!string.IsNullOrEmpty(delete)) + { + Program.Console($"Deleting from {collectionName}: {delete}"); + var deleteResult = MongoDatabase.GetCollection(collectionName).DeleteMany(delete); + Program.Console($"Deleted from {collectionName}: {deleteResult.DeletedCount} documents deleted"); + } + } + + } + + private void dropCollection(string collectionName) + { + Program.Console($"Dropping collection {collectionName}"); + MongoDatabase.DropCollection(collectionName); + Program.Console($"Collection {collectionName} dropped"); + + } + + public const string IMPORT_COMMAND = "mongoimport"; + public const string MONGO_COMMAND = "mongo"; + public const string SyncCollectionName = "_mongoSync"; + + private const string CREATE_INDEX = "var r = db.{0}.createIndex({1}); if(r.ok!=1) {{quit(r.code)}}"; + + public Func> ReadCompletedImports { get; set; } + public Func ReadFiles { get; set; } = (fileFilter) => new DirectoryInfo(".").GetFiles(fileFilter); + public Func ConnectToMongoDb { get; set; } + + public IMongoDatabase MongoDatabase { get; private set; } + public IMongoCollection SyncCollection { get; private set; } + + // Wrapping intp function to override for testing, no plan to unit test this + public Dictionary ReadCompletedImportsDefault() + { + Program.Console($"Reading information about completed imports from {SyncCollectionName}"); + var completedImports = SyncCollection.Find((f) => f.Success).ToEnumerable(); + var result = completedImports.ToDictionary(e => e.FileName, e => e); + Program.Console($"{result.Count} imported items found"); + return result; + } + + protected override Dictionary> ParsingRules() + { + var result = base.ParsingRules(); + result.Add("--file", (cmd, arg) => (cmd as ImportCommand).FilesFilter = arg); + return result; + } + + public override void Validate() + { + if (string.IsNullOrEmpty(Connection)) + { + throw new ArgumentException("Connection parameter required for import command"); + } + if (string.IsNullOrEmpty(FilesFilter)) + { + throw new ArgumentException("Connection parameter required for import command"); + } + ConnectionString = new ConnectionString(Connection); + + // try to connect + MongoDatabase = ConnectToMongoDb(Connection, ConnectionString.Database); + SyncCollection = MongoDatabase.GetCollection(SyncCollectionName); + } + + private IMongoDatabase ConnectToMongoDbDefault(string connection, string databaseName) + { + var mongoClient = new MongoClient(Connection); + return mongoClient.GetDatabase(ConnectionString.Database); + } + } +} diff --git a/SocialTalents.MongoSync.Console/Model/ImportMode.cs b/SocialTalents.MongoSync.Console/Model/ImportMode.cs new file mode 100644 index 0000000..ab9c4f9 --- /dev/null +++ b/SocialTalents.MongoSync.Console/Model/ImportMode.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace SocialTalents.MongoSync.Console.Model +{ + public enum ImportMode + { + Unknown = 0, + Insert, + Upsert, + Merge, + Delete, + Drop, + Eval, + CreateIndex + } +} diff --git a/SocialTalents.MongoSync.Console/Model/SyncEntity.cs b/SocialTalents.MongoSync.Console/Model/SyncEntity.cs new file mode 100644 index 0000000..06c09de --- /dev/null +++ b/SocialTalents.MongoSync.Console/Model/SyncEntity.cs @@ -0,0 +1,17 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using System; +using System.Collections.Generic; +using System.Text; + +namespace SocialTalents.MongoSync.Console.Model +{ + public class SyncEntity + { + public ObjectId Id { get; set; } = ObjectId.GenerateNewId(); + public string FileName { get; set; } + [BsonDateTimeOptions(Kind = DateTimeKind.Utc)] + public DateTime Imported { get; set; } = DateTime.UtcNow; + public bool Success { get; set; } = false; + } +} diff --git a/SocialTalents.MongoSync.Console/MongoSync.bat b/SocialTalents.MongoSync.Console/MongoSync.bat new file mode 100644 index 0000000..1bca57e --- /dev/null +++ b/SocialTalents.MongoSync.Console/MongoSync.bat @@ -0,0 +1,2 @@ +SET PATH=%PATH%;C:\Program Files\MongoDB\Server\3.6\bin\; +dotnet MongoSync.suo/SocialTalents.MongoSync.Console.dll %* \ No newline at end of file diff --git a/SocialTalents.MongoSync.Console/MongoSync.sh b/SocialTalents.MongoSync.Console/MongoSync.sh new file mode 100644 index 0000000..816d7fb --- /dev/null +++ b/SocialTalents.MongoSync.Console/MongoSync.sh @@ -0,0 +1 @@ +dotnet bin/SocialTalents.MongoSync.Console.dll "$@" \ No newline at end of file diff --git a/SocialTalents.MongoSync.Console/Program.cs b/SocialTalents.MongoSync.Console/Program.cs new file mode 100644 index 0000000..ded1f01 --- /dev/null +++ b/SocialTalents.MongoSync.Console/Program.cs @@ -0,0 +1,85 @@ +using SocialTalents.MongoSync.Console.Model; +using System; +using System.Linq; +using System.Diagnostics; + +namespace SocialTalents.MongoSync.Console +{ + public class Program + { + public static void Main(string[] args) + { + Command cmd = ParseCommand(args); + cmd.Validate(); + + cmd.Execute(); + } + + public static Action Console = (s) => System.Console.WriteLine(s); + public static Func Exec = (cmd, arg) => + { + Console("executing mongo command..."); + Console($"{cmd} {arg}"); + + ProcessStartInfo startInfo = new ProcessStartInfo(cmd, arg); + startInfo.RedirectStandardOutput = true; + startInfo.RedirectStandardError = true; + startInfo.UseShellExecute = false; + startInfo.CreateNoWindow = true; + + Process proc = new Process(); + proc.StartInfo = startInfo; + proc.Start(); + + proc.WaitForExit(); + + var output = proc.StandardOutput.ReadToEnd(); + var error = proc.StandardError.ReadToEnd(); + + Console($"Execution completed with state {proc.ExitCode}"); + Console(output); + Console(error); + return proc.ExitCode; + }; + + public static Command ParseCommand(params string[] args) + { + Command result = new HelpCommand(); + // try parse arguments only when they exist + if (args != null && args.Length > 0) + { + try + { + Command candidate = new HelpCommand(); + CommandType cmdType = Enum.Parse(args[0], true); + switch (cmdType) + { + case CommandType.Help: + candidate = new HelpCommand(); + break; + case CommandType.Import: + candidate = new ImportCommand(); + break; + case CommandType.Export: + candidate = new ExportCommand(); + break; + default: throw new NotImplementedException($"Command {cmdType} not implemented yet"); + } + candidate.CommandType = cmdType; + candidate.Parse(args); + result = candidate; + } + catch (Exception ex) + { + Console(ex.Message); +#if DEBUG + Console(ex.StackTrace); +#endif + } + } + return result; + } + + + } +} diff --git a/SocialTalents.MongoSync.Console/Properties/launchSettings.json b/SocialTalents.MongoSync.Console/Properties/launchSettings.json new file mode 100644 index 0000000..888d905 --- /dev/null +++ b/SocialTalents.MongoSync.Console/Properties/launchSettings.json @@ -0,0 +1,7 @@ +{ + "profiles": { + "SocialTalents.MongoSync.Console": { + "commandName": "Project" + } + } +} \ No newline at end of file diff --git a/SocialTalents.MongoSync.Console/Properties/launchSettings.vk.json b/SocialTalents.MongoSync.Console/Properties/launchSettings.vk.json new file mode 100644 index 0000000..e37f234 --- /dev/null +++ b/SocialTalents.MongoSync.Console/Properties/launchSettings.vk.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "SocialTalents.MongoSync.Console": { + "commandName": "Project", + "commandLineArgs": "import --conn mongodb://127.0.0.1:27027/SkyHiveV3" + } + } +} \ No newline at end of file diff --git a/SocialTalents.MongoSync.Console/SocialTalents.MongoSync.Console.csproj b/SocialTalents.MongoSync.Console/SocialTalents.MongoSync.Console.csproj new file mode 100644 index 0000000..f170abf --- /dev/null +++ b/SocialTalents.MongoSync.Console/SocialTalents.MongoSync.Console.csproj @@ -0,0 +1,27 @@ + + + + Exe + netcoreapp2.0 + 1.2.0.0 + + + + + + + + + + Always + + + Always + + + + + + + + diff --git a/SocialTalents.MongoSync.Console/debugrun.bat b/SocialTalents.MongoSync.Console/debugrun.bat new file mode 100644 index 0000000..6e341e5 --- /dev/null +++ b/SocialTalents.MongoSync.Console/debugrun.bat @@ -0,0 +1,2 @@ +SET PATH=%PATH%;C:\Program Files\MongoDB\Server\3.6\bin\; +dotnet run %* \ No newline at end of file diff --git a/SocialTalents.MongoSync.Console/publish.bat b/SocialTalents.MongoSync.Console/publish.bat new file mode 100644 index 0000000..db1d54f --- /dev/null +++ b/SocialTalents.MongoSync.Console/publish.bat @@ -0,0 +1,2 @@ +dotnet publish -c Release +powershell -Command "Compress-Archive -Path bin\release\netcoreapp2.0\publish\* -DestinationPath publish\latest.zip -Force" \ No newline at end of file diff --git a/SocialTalents.MongoSync.XUnit/ConnectionStringTest.cs b/SocialTalents.MongoSync.XUnit/ConnectionStringTest.cs new file mode 100644 index 0000000..b971558 --- /dev/null +++ b/SocialTalents.MongoSync.XUnit/ConnectionStringTest.cs @@ -0,0 +1,49 @@ +using SocialTalents.MongoSync.Console.Model; +using System; +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace SocialTalents.MongoSync.XUnit +{ + public class ConnectionStringTest + { + // mongo.exe requires database name as first parameter, in this way it is easier to fix parameters + [Theory] + [InlineData("mongodb://masterDev:BfkvYB@127.0.0.1/master", new []{"127.0.0.1:27017"}, "masterDev", "BfkvYB", "master", + "--db master --host 127.0.0.1:27017 --username masterDev --password BfkvYB")] + [InlineData("mongodb://masterDev@127.0.0.1:28017/master", new []{"127.0.0.1:28017"}, "masterDev", null, "master", + "--db master --host 127.0.0.1:28017 --username masterDev")] + [InlineData("mongodb://127.0.0.1:28017/master", new []{"127.0.0.1:28017"}, null, null, "master" + , "--db master --host 127.0.0.1:28017")] + [InlineData("mongodb://masterDev:BfkvYB@127.0.0.1:28017/master", new []{"127.0.0.1:28017"}, "masterDev", "BfkvYB", "master" + , "--db master --host 127.0.0.1:28017 --username masterDev --password BfkvYB")] + // takes first host from the list if not replica set specified: + [InlineData("mongodb://127.0.0.1:28018,127.0.0.1:28019/master", new []{"127.0.0.1:28018", "127.0.0.1:28019"}, null, null, "master" + , "--db master --host 127.0.0.1:28018")] + // parses replica set: + [InlineData("mongodb://127.0.0.1:28018,127.0.0.1:28019/master?replicaSet=rs0", new []{"127.0.0.1:28018", "127.0.0.1:28019"}, null, null, "master" + , "--db master --host rs0/127.0.0.1:28018,127.0.0.1:28019")] + // ssl on + [InlineData("mongodb://127.0.0.1:28018,127.0.0.1:28019/master?replicaSet=rs0&ssl=true", new[] { "127.0.0.1:28018", "127.0.0.1:28019" }, null, null, "master" + , "--db master --host rs0/127.0.0.1:28018,127.0.0.1:28019 --ssl")] + public void TestParsing(string input, string[] hosts, string user, string password, string database, string expectedCommandLine) + { + var c = new ConnectionString(input); + + Assert.Equal(hosts, c.Hosts); + Assert.Equal(user, c.UserName); + Assert.Equal(password, c.Password); + Assert.Equal(database, c.Database); + + Assert.Equal(expectedCommandLine, c.ToCommandLine()); + } + + [Fact] + public void DbNameRequired() + { + string connectionWithoutDbName = "masterDev:BfkvYB@127.0.0.1:28017"; + Assert.Throws(() => new ConnectionString(connectionWithoutDbName)); + } + } +} diff --git a/SocialTalents.MongoSync.XUnit/ImportCommandTest.cs b/SocialTalents.MongoSync.XUnit/ImportCommandTest.cs new file mode 100644 index 0000000..95935fe --- /dev/null +++ b/SocialTalents.MongoSync.XUnit/ImportCommandTest.cs @@ -0,0 +1,74 @@ +using MongoDB.Driver; +using Moq; +using SocialTalents.MongoSync.Console; +using SocialTalents.MongoSync.Console.Model; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using Xunit; + +namespace SocialTalents.MongoSync.XUnit +{ + public class ImportCommandTest + { + ImportCommand _command; + StringBuilder _execLog = new StringBuilder(); + StringBuilder _consoleLog = new StringBuilder(); + Mock _mongoDatabaseMock = new Mock(); + Mock> _syncEntityCollection = new Mock>(); + + public ImportCommandTest() + { + _command = new ImportCommand(); + _command.Connection = "mongodb://localhost/db"; + _command.ConnectToMongoDb = (conn, db) => _mongoDatabaseMock.Object; + + _mongoDatabaseMock.Setup(db => db.GetCollection(ImportCommand.SyncCollectionName, null)).Returns(_syncEntityCollection.Object); + _mongoDatabaseMock.Setup(db => db.DropCollection("Collection", It.IsAny())).Verifiable(); + + Program.Exec = (cmd, arg) => { _execLog.AppendLine($"{cmd} {arg}"); return arg.Contains("err") ? 1 : 0; }; + Program.Console = (line) => _consoleLog.AppendLine(line); + } + + [Fact] + public void FilesSetProperly() + { + Assert.Equal("*.js*", _command.FilesFilter); + } + + [Fact] + public void HappyIntegration() + { + _command.ReadCompletedImports = () => new Dictionary() { { "123.Collection.Insert.json", new SyncEntity() { Imported = DateTime.Now } } }; + _command.ReadFiles = (filter) => new FileInfo[] { new FileInfo("123.Collection.Insert.json"), new FileInfo("234.Collection.Insert.json"), new FileInfo("123.Collection.drop.json"), + new FileInfo("345.Collection.eval.js")}; + _command.Validate(); + _command.Execute(); + + _mongoDatabaseMock.Verify(); + string fullLog = _consoleLog.ToString(); + // Error message not found + Assert.True(fullLog.IndexOf("Error") < 0); + + /* +Found 2 files to import +Importing 123.Collection.drop.json to collection Collection, mode Drop +Dropping collection Collection +Collection Collection dropped +Importing 234.Collection.Insert.json to collection Collection, mode Insert +Import completed successfully + */ + + Assert.Contains("Found 4 files to import", fullLog); + Assert.Contains("Skipping 123.Collection.Insert.json", fullLog); + Assert.Contains("Dropping collection Collection", fullLog); + Assert.Contains("Importing 234.Collection.Insert.json to collection Collection, mode Insert", fullLog); + Assert.Contains("Importing 345.Collection.eval.js to collection Collection, mode Eval", fullLog); + Assert.Contains("successfully", fullLog); + } + + + } +} diff --git a/SocialTalents.MongoSync.XUnit/ProgramTests.cs b/SocialTalents.MongoSync.XUnit/ProgramTests.cs new file mode 100644 index 0000000..ec4085a --- /dev/null +++ b/SocialTalents.MongoSync.XUnit/ProgramTests.cs @@ -0,0 +1,109 @@ +using SocialTalents.MongoSync.Console; +using SocialTalents.MongoSync.Console.Model; +using System; +using System.Text; +using Xunit; + +namespace SocialTalents.MongoSync.XUnit +{ + public class ProgramTests + { + StringBuilder _output = new StringBuilder(); + public ProgramTests() + { + Program.Console = (s) => _output.AppendLine(s); + } + + [Fact] + public void NoParams_NoError() + { + Program.Main(new string[0]); + Assert.True(_output.ToString().IndexOf("Usage") == 0); + } + + [Fact] + public void HelpCommand_Lowercase() + { + Program.Main(new string[] { "help"}); + Assert.True(_output.ToString().IndexOf("Usage") == 0); + } + + [Theory] + [InlineData("help", typeof(HelpCommand), CommandType.Help)] + [InlineData("IMPORT", typeof(ImportCommand), CommandType.Import)] + [InlineData("import", typeof(ImportCommand), CommandType.Import)] + [InlineData("eXport", typeof(ExportCommand), CommandType.Export)] + public void ParseCommand_Type(string commandName, Type t, CommandType type) + { + Command c = Program.ParseCommand(commandName); + Assert.Equal(type, c.CommandType); + Assert.IsType(t, c); + } + + [Fact] + // Local mongodb must be accessble for this test to succeed + public void ValidImport() + { + Command c = new ImportCommand(); + c.Parse("--uri mongodb://localhost:27017/test --file *.json".Split(' ')); + Assert.Equal("mongodb://localhost:27017/test", c.Connection); + Assert.Equal("*.json", (c as ImportCommand).FilesFilter); + c.Validate(); + } + + [Fact] + public void ValidExport() + { + var c = new ExportCommand(); + c.Parse("--uri mongocon --query {} --collection countries".Split(' ')); + Assert.Equal("mongocon", c.Connection); + Assert.Equal("{}", c.SearchQueryForExport); + Assert.Equal("countries", (c as ExportCommand).CollectionName); + c.Validate(); + } + + [Fact] + public void Export_Execute_Atlas() + { + string executable = null; + string arguments = null; + Program.Exec = (cmd, args) => { executable = cmd; arguments = args; return 127; }; + var c = new ExportCommand(); + c.Parse("--uri mongodb://user:password@host:28123,host2:28125/database?replicaSet=Atlas1-shard-0&ssl=true true --collection Countries --query {a:1} --authenticationDatabase admin".Split(' ')); + c.Execute(); + + Assert.Equal(ExportCommand.COMMAND, executable); + Assert.Equal($"--db database --host Atlas1-shard-0/host:28123,host2:28125 --username user --password password --ssl --authenticationDatabase admin --collection Countries --query {{a:1}} --type json --out {c.TimePrefix}.Countries.Insert.json", arguments); + } + + [Fact] + public void Export_Execute_Simple() + { + string executable = null; + string arguments = null; + Program.Exec = (cmd, args) => { executable = cmd; arguments = args; return 127; }; + var c = new ExportCommand(); + c.Parse("--uri mongodb://host:28123/database --collection Countries --query {a:1}".Split(' ')); + c.Execute(); + + Assert.Equal(ExportCommand.COMMAND, executable); + Assert.Equal($"--db database --host host:28123 --collection Countries --query {{a:1}} --type json --out {c.TimePrefix}.Countries.Insert.json", arguments); + } + + [Fact] + public void NotValidExport_CollectionMissing() + { + var c = new ExportCommand(); + c.Parse("--uri mongocon --query {}".Split(' ')); + Assert.Equal("mongocon", c.Connection); + Assert.Equal("{}", c.SearchQueryForExport); + Assert.Throws(() => c.Validate()); + } + + [Fact] + public void DefaultCommand_IsValid_RequiresImplementation() + { + Assert.Throws(() => new Command().Validate()); + } + } +} diff --git a/SocialTalents.MongoSync.XUnit/SocialTalents.MongoSync.XUnit.csproj b/SocialTalents.MongoSync.XUnit/SocialTalents.MongoSync.XUnit.csproj new file mode 100644 index 0000000..78fac86 --- /dev/null +++ b/SocialTalents.MongoSync.XUnit/SocialTalents.MongoSync.XUnit.csproj @@ -0,0 +1,26 @@ + + + + netcoreapp2.0 + + false + + + + + + + + + + + + + + + + + + + + diff --git a/install_linux.sh b/install_linux.sh new file mode 100644 index 0000000..0b5c003 --- /dev/null +++ b/install_linux.sh @@ -0,0 +1,5 @@ +wget https://github.com/Socialtalents/SocialTalents.MongoSync/raw/master/SocialTalents.MongoSync.Console/Publish/Release.zip +# you might need to install unzip +# sudo apt-get install unzip +unzip Release.zip -d bin/ +cp bin/MongoSync.sh ./mongosync.sh diff --git a/install_windows.bat b/install_windows.bat new file mode 100644 index 0000000..0720edc --- /dev/null +++ b/install_windows.bat @@ -0,0 +1,7 @@ +@Echo off +powershell -Command "Invoke-WebRequest https://github.com/Socialtalents/SocialTalents.MongoSync/raw/master/SocialTalents.MongoSync.Console/Publish/Release.zip -OutFile MongoSync.zip" +Echo Installing MongoSync to MongoSync.suo so it will be invisible for git +powershell -Command "Expand-Archive MongoSync.zip -DestinationPath MongoSync.suo -Force" +copy MongoSync.suo\MongoSync.bat . +del MongoSync.zip +