From fa94ecbb5a0488218ddbe2a631a0a05d99e7b178 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 24 Sep 2013 14:15:58 +0100 Subject: [PATCH 01/28] Updated to latest NPM packages. Added simple mocha for testing purposes --- package.json | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index eb2a7c7..2d46a6d 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,14 @@ "test": "grunt test" }, "dependencies": { - "shelljs": "~0.1.4" + "shelljs": "~0.2.6" }, "devDependencies": { - "grunt-contrib-jshint": "~0.1.1", - "grunt-contrib-clean": "~0.4.0", + "grunt-contrib-jshint": "~0.6.4", + "grunt-contrib-clean": "~0.5.0", "grunt-contrib-nodeunit": "~0.1.2", - "grunt": "~0.4.1" + "grunt": "~0.4.1", + "grunt-simple-mocha": "~0.4.0" }, "peerDependencies": { "grunt": "~0.4.1" @@ -48,4 +49,4 @@ "mysql", "wordpress" ] -} \ No newline at end of file +} From 9e87994fd4dfba506bf5ccb8a1b3fb512a8d5f97 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 25 Sep 2013 12:47:42 +0100 Subject: [PATCH 02/28] Integrate Chai expect style assertions. Add x2 dummy tests --- Gruntfile.js | 27 ++++++++++--------- package.json | 4 ++- test/deployments_test.js | 58 +++++++++------------------------------- 3 files changed, 30 insertions(+), 59 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index e364a61..e6913d9 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -10,6 +10,8 @@ module.exports = function(grunt) { + // load all grunt tasks + require('matchdep').filterDev('grunt-*').forEach(grunt.loadNpmTasks); // Project configuration. grunt.initConfig({ @@ -31,33 +33,34 @@ module.exports = function(grunt) { tests: ['tmp'], }, - deployments: { + simplemocha: { options: { - backups_dir: '' + globals: ['chai'], + reporter: 'Nyan' }, - local: '<%= db_fixture.local %>', - develop: '<%= db_fixture.develop %>' - }, - // Unit tests. - nodeunit: { - tests: ['test/*_test.js'], - }, + all: { + src: [ + 'test/**/*.js' + ] + } + } }); - // Actually load this plugin's task(s). +/* // Actually load this plugin's task(s). grunt.loadTasks('tasks'); // These plugins provide necessary tasks. grunt.loadNpmTasks('grunt-contrib-jshint'); grunt.loadNpmTasks('grunt-contrib-clean'); - grunt.loadNpmTasks('grunt-contrib-nodeunit'); + grunt.loadNpmTasks('chai'); + grunt.loadNpmTasks('grunt-simple-mocha'); */ // Whenever the "test" task is run, first clean the "tmp" dir, then run this // plugin's task(s), then test the result. - grunt.registerTask('test', ['clean', 'deployments', 'nodeunit']); + grunt.registerTask('test', ['clean', 'simplemocha']); // By default, lint and run all tests. grunt.registerTask('default', ['jshint', 'test']); diff --git a/package.json b/package.json index 2d46a6d..fef79f2 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,9 @@ "grunt-contrib-clean": "~0.5.0", "grunt-contrib-nodeunit": "~0.1.2", "grunt": "~0.4.1", - "grunt-simple-mocha": "~0.4.0" + "grunt-simple-mocha": "~0.4.0", + "chai": "~1.8.0", + "matchdep": "~0.1.2" }, "peerDependencies": { "grunt": "~0.4.1" diff --git a/test/deployments_test.js b/test/deployments_test.js index 0632799..43adfae 100644 --- a/test/deployments_test.js +++ b/test/deployments_test.js @@ -1,48 +1,14 @@ 'use strict'; -var grunt = require('grunt'); - -/* - ======== A Handy Little Nodeunit Reference ======== - https://github.com/caolan/nodeunit - - Test methods: - test.expect(numAssertions) - test.done() - Test assertions: - test.ok(value, [message]) - test.equal(actual, expected, [message]) - test.notEqual(actual, expected, [message]) - test.deepEqual(actual, expected, [message]) - test.notDeepEqual(actual, expected, [message]) - test.strictEqual(actual, expected, [message]) - test.notStrictEqual(actual, expected, [message]) - test.throws(block, [error], [message]) - test.doesNotThrow(block, [error], [message]) - test.ifError(value) -*/ - -exports.deployments = { - setUp: function(done) { - // setup here if necessary - done(); - }, - default_options: function(test) { - test.expect(1); - - var actual = grunt.file.read('tmp/default_options'); - var expected = grunt.file.read('test/expected/default_options'); - test.equal(actual, expected, 'should describe what the default behavior is.'); - - test.done(); - }, - custom_options: function(test) { - test.expect(1); - - var actual = grunt.file.read('tmp/custom_options'); - var expected = grunt.file.read('test/expected/custom_options'); - test.equal(actual, expected, 'should describe what the custom option(s) behavior is.'); - - test.done(); - }, -}; +var chai = require("chai"); +var expect = require('chai').expect + + +describe('Array', function(){ + describe('#indexOf()', function(){ + it('should check that a number is equal to another number', function(){ + expect(1).to.equal(1); + expect(3).to.equal(3); + }) + }) +}); From 361d2bbea73f5479fdcfc0a913dd57f45e8dfac7 Mon Sep 17 00:00:00 2001 From: David Date: Wed, 25 Sep 2013 13:45:22 +0100 Subject: [PATCH 03/28] Experiment with adding and interacting with Grunt via tests --- Gruntfile.js | 15 ++++++--------- test/deployments_test.js | 31 ++++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index e6913d9..3126c8a 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -35,29 +35,26 @@ module.exports = function(grunt) { simplemocha: { options: { - globals: ['chai'], - reporter: 'Nyan' + globals: ['chai'] }, all: { src: [ + 'node_modules/chai/lib/chai.js', 'test/**/*.js' ] } + }, + deployments: { + } }); -/* // Actually load this plugin's task(s). +// Actually load this plugin's task(s). grunt.loadTasks('tasks'); - // These plugins provide necessary tasks. - grunt.loadNpmTasks('grunt-contrib-jshint'); - grunt.loadNpmTasks('grunt-contrib-clean'); - grunt.loadNpmTasks('chai'); - grunt.loadNpmTasks('grunt-simple-mocha'); */ - // Whenever the "test" task is run, first clean the "tmp" dir, then run this // plugin's task(s), then test the result. grunt.registerTask('test', ['clean', 'simplemocha']); diff --git a/test/deployments_test.js b/test/deployments_test.js index 43adfae..08e8065 100644 --- a/test/deployments_test.js +++ b/test/deployments_test.js @@ -1,14 +1,27 @@ 'use strict'; -var chai = require("chai"); -var expect = require('chai').expect -describe('Array', function(){ - describe('#indexOf()', function(){ - it('should check that a number is equal to another number', function(){ - expect(1).to.equal(1); - expect(3).to.equal(3); - }) - }) +var chai = require("chai"); +var expect = require('chai').expect; + +var grunt = require('grunt'); +var deployments = require('../tasks/deployments.js'); + + + +describe('Check Grunt Registration', function(){ + it('registers deployments task with Grunt', function(){ + expect(deployments).to.exist; + expect(deployments).to.be.a('function'); + }); + + it('registers db_pull task with Grunt', function(){ + expect(grunt.task._tasks['db_pull']).to.exist; + }); + + it('registers db_push task with Grunt', function(){ + expect(grunt.task._tasks['db_push']).to.exist; + }); }); + From 5fe4e4ffc34156bcf1dc924f3fc8395fa0199427 Mon Sep 17 00:00:00 2001 From: Dan Bissonnet Date: Mon, 25 Nov 2013 10:22:35 +0000 Subject: [PATCH 04/28] Add support for custom MySQL ports --- tasks/deployments.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tasks/deployments.js b/tasks/deployments.js index 38b2ceb..c690568 100644 --- a/tasks/deployments.js +++ b/tasks/deployments.js @@ -142,7 +142,8 @@ module.exports = function(grunt) { user: config.user, pass: config.pass, database: config.database, - path: src + path: src, + port: config.port || 3306 } }); @@ -187,7 +188,8 @@ module.exports = function(grunt) { user: config.user, pass: config.pass, database: config.database, - host: config.host + host: config.host, + port: config.port || 3306 } }); @@ -249,9 +251,9 @@ module.exports = function(grunt) { search_replace: "sed -i '' 's#<%= search %>#<%= replace %>#g' <%= path %>", - mysqldump: "mysqldump -h <%= host %> -u<%= user %> -p<%= pass %> <%= database %>", + mysqldump: "mysqldump -h <%= host %> -u<%= user %> -p<%= pass %> -P<%= port %> <%= database %>", - mysql: "mysql -h <%= host %> -u <%= user %> -p<%= pass %> <%= database %>", + mysql: "mysql -h <%= host %> -u <%= user %> -p<%= pass %> -P<%= port %> <%= database %>", ssh: "ssh <%= host %>", }; From aebffa47363715347951500c5540d00a8eb102cc Mon Sep 17 00:00:00 2001 From: Dan Bissonnet Date: Mon, 25 Nov 2013 10:26:57 +0000 Subject: [PATCH 05/28] Update README.md with new database port option --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5bf4625..68e0312 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,10 @@ Description: the password for the database user (above) Type: `String` Description: the hostname for the location in which the database resides. Typically this will be `localhost` +#### port +Type: `Integer` +Description: the port that MySQL is running on. Defaults to `3306` + #### url Type: `String` Description: the string to search and replace within the database before it is moved to the target location. Typically this is designed for use with systems such as WordPress where the `siteurl` value is [stored in the database](http://codex.wordpress.org/Changing_The_Site_URL) and is required to be updated upon migration to a new environment. It is however suitable for replacing any single value within the database before it is moved. From 41e764bfc6cd894ef40c31b856eccae974793bbd Mon Sep 17 00:00:00 2001 From: Nico Prat Date: Mon, 9 Dec 2013 18:28:51 +0100 Subject: [PATCH 06/28] Added 'ignoreTables' option (ref #24) Also updated Readme and bumped version number. --- README.md | 29 +++++++++++++++++++++-------- tasks/deployments.js | 11 ++++++++--- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 5bf4625..34a2df7 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,8 @@ As a result, it is essential that you define a *single* target *without* an `ssh "user": "local_db_username", "pass": "local_db_password", "host": "local_db_host", - "url": "local_db_url" + "url": "local_db_url", + "ignoreTables": ["table1","table2",...] // note that the `local` target does not have an "ssh_host" }, ``` @@ -113,7 +114,8 @@ All other targets *must* contain a valid `ssh_host` parameter. "pass": "development_db_password", "host": "development_db_host", "url": "development_db_url", - "ssh_host": "ssh_user@ssh_host" + "ssh_host": "ssh_user@ssh_host", + "ignoreTables": ["table1","table2",...] }, "stage": { "title": "Stage", @@ -122,7 +124,8 @@ All other targets *must* contain a valid `ssh_host` parameter. "pass": "stage_db_password", "host": "stage_db_host", "url": "stage_db_url", - "ssh_host": "ssh_user@ssh_host" + "ssh_host": "ssh_user@ssh_host", + "ignoreTables": ["table1","table2",...] }, "production": { "title": "Production", @@ -131,7 +134,8 @@ All other targets *must* contain a valid `ssh_host` parameter. "pass": "production_db_password", "host": "production_db_host", "url": "production_db_url", - "ssh_host": "ssh_user@ssh_host" + "ssh_host": "ssh_user@ssh_host", + "ignoreTables": ["table1","table2",...] } ``` @@ -152,7 +156,8 @@ grunt.initConfig({ "user": "local_db_username", "pass": "local_db_password", "host": "local_db_host", - "url": "local_db_url" + "url": "local_db_url", + "ignoreTables": ["table1","table2",...] // note that the `local` target does not have an "ssh_host" }, // "Remote" targets @@ -163,7 +168,8 @@ grunt.initConfig({ "pass": "development_db_password", "host": "development_db_host", "url": "development_db_url", - "ssh_host": "ssh_user@ssh_host" + "ssh_host": "ssh_user@ssh_host", + "ignoreTables": ["table1","table2",...] }, "stage": { "title": "Stage", @@ -172,7 +178,8 @@ grunt.initConfig({ "pass": "stage_db_password", "host": "stage_db_host", "url": "stage_db_url", - "ssh_host": "ssh_user@ssh_host" + "ssh_host": "ssh_user@ssh_host", + "ignoreTables": ["table1","table2",...] }, "production": { "title": "Production", @@ -181,7 +188,8 @@ grunt.initConfig({ "pass": "production_db_password", "host": "production_db_host", "url": "production_db_url", - "ssh_host": "ssh_user@ssh_host" + "ssh_host": "ssh_user@ssh_host", + "ignoreTables": ["table1","table2",...] } }, }) @@ -219,6 +227,10 @@ Description: the string to search and replace within the database before it is m Type: `String` Description: ssh connection string in the format `SSH_USER@SSH_HOST`. The task assumes you have ssh keys setup which allow you to remote into your server without requiring the input of a password. As this is an exhaustive topic we will not cover it here but you might like to start by reading [Github's own advice](https://help.github.com/articles/generating-ssh-keys). +#### ignoreTables +Type: `Array of Strings` +Description: tables to ignore. They won't be in the dump — neither their structure nor their content. + ### Options #### options.backups_dir @@ -244,6 +256,7 @@ In lieu of a formal styleguide, take care to maintain the existing coding style. ## Release History +* 2013-12-09 v0.3.0 Added `ignoreTables` option. * 2013-11-12   v0.2.0   Fix escaping issues, ability to define `target` via options, README doc fixes, pass host param to mysqldump. * 2013-06-11   v0.1.0   Minor updates to docs including addtion of Release History section. * 2013-06-11   v0.0.1   Initial Plugin release. diff --git a/tasks/deployments.js b/tasks/deployments.js index 38b2ceb..2bb406f 100644 --- a/tasks/deployments.js +++ b/tasks/deployments.js @@ -180,6 +180,10 @@ module.exports = function(grunt) { grunt.file.mkdir(output_paths.dir); + // 1) Get array of tables to ignore, as defined in the config, and format it correctly + if( config.ignoreTables ) { + var ignoreTables = '--ignore-table=' + config.database + "." + config.ignoreTables.join(' --ignore-table='+config.database+'.'); + } // 2) Compile MYSQL cmd via Lo-Dash template string var tpl_mysqldump = grunt.template.process(tpls.mysqldump, { @@ -187,12 +191,13 @@ module.exports = function(grunt) { user: config.user, pass: config.pass, database: config.database, - host: config.host + host: config.host, + ignoreTables: ignoreTables || '' } }); - // 3) Test whether MYSQL DB is local or whether requires remote access via SSH + if (typeof config.ssh_host === "undefined") { // it's a local connection grunt.log.writeln("Creating DUMP of local database"); cmd = tpl_mysqldump; @@ -249,7 +254,7 @@ module.exports = function(grunt) { search_replace: "sed -i '' 's#<%= search %>#<%= replace %>#g' <%= path %>", - mysqldump: "mysqldump -h <%= host %> -u<%= user %> -p<%= pass %> <%= database %>", + mysqldump: "mysqldump -h <%= host %> -u<%= user %> -p<%= pass %> <%= database %> <%= ignoreTables %>", mysql: "mysql -h <%= host %> -u <%= user %> -p<%= pass %> <%= database %>", From 66e80ace2e6abd8608cf12183d081abf5ae7583e Mon Sep 17 00:00:00 2001 From: David Date: Thu, 2 Jan 2014 11:43:21 +0000 Subject: [PATCH 07/28] Update dependencies. Add vows for tests. Fix JSHint warnings. --- .jshintrc | 3 +- Gruntfile.js | 83 ++++++++++++++++++++++++++++------------ package.json | 12 +++--- tasks/deployments.js | 3 +- test/deployments_test.js | 4 -- 5 files changed, 68 insertions(+), 37 deletions(-) diff --git a/.jshintrc b/.jshintrc index 6b4c1a9..f57a8ff 100644 --- a/.jshintrc +++ b/.jshintrc @@ -9,6 +9,5 @@ "undef": true, "boss": true, "eqnull": true, - "node": true, - "es5": true + "node": true } diff --git a/Gruntfile.js b/Gruntfile.js index 3126c8a..8bea82d 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -2,12 +2,10 @@ * grunt-deployments * https://github.com/getdave/grunt-deployments * - * Copyright (c) 2013 David Smith + * Copyright (c) 2014 David Smith * Licensed under the MIT license. */ -'use strict'; - module.exports = function(grunt) { // load all grunt tasks @@ -16,37 +14,74 @@ module.exports = function(grunt) { // Project configuration. grunt.initConfig({ db_fixture: grunt.file.readJSON('test/fixtures/test_db.json'), - + vows: { + all: { + options: { + // String {spec|json|dot-matrix|xunit|tap} + // defaults to "dot-matrix" + reporter: "spec", + // String or RegExp which is + // matched against title to + // restrict which tests to run + // onlyRun: /helper/, + // Boolean, defaults to false + verbose: false, + // Boolean, defaults to false + silent: false, + // Colorize reporter output, + // boolean, defaults to true + colors: true, + // Run each test in its own + // vows process, defaults to + // false + isolate: false, + // String {plain|html|json|xml} + // defaults to none + coverage: "json" + }, + // String or array of strings + // determining which files to include. + // This option is grunt's "full" file format. + src: ["test/**/*_test.js"] + } + }, jshint: { - all: [ - 'Gruntfile.js', - 'tasks/*.js', - '<%= nodeunit.tests %>', - ], + gruntfile: { + src: 'Gruntfile.js' + }, + lib: { + src: ['tasks/*.js'] + }, + test: { + src: ['test/**/*.js'] + }, options: { jshintrc: '.jshintrc', }, }, + watch: { + gruntfile: { + files: '<%= jshint.gruntfile.src %>', + tasks: ['jshint:gruntfile'] + }, + lib: { + files: '<%= jshint.lib.src %>', + tasks: ['jshint:lib', 'vows'] + }, + test: { + files: '<%= jshint.test.src %>', + tasks: ['jshint:test', 'vows'] + } + }, + // Before generating any new files, remove any previously-created files. clean: { tests: ['tmp'], }, - simplemocha: { - options: { - globals: ['chai'] - }, - - all: { - src: [ - 'node_modules/chai/lib/chai.js', - 'test/**/*.js' - ] - } - }, deployments: { - + } }); @@ -57,9 +92,9 @@ module.exports = function(grunt) { // Whenever the "test" task is run, first clean the "tmp" dir, then run this // plugin's task(s), then test the result. - grunt.registerTask('test', ['clean', 'simplemocha']); + grunt.registerTask('test', ['clean', 'vows']); // By default, lint and run all tests. - grunt.registerTask('default', ['jshint', 'test']); + grunt.registerTask('default', ['jshint', 'vows']); }; diff --git a/package.json b/package.json index ff1f124..b434083 100644 --- a/package.json +++ b/package.json @@ -32,13 +32,13 @@ "shelljs": "~0.2.6" }, "devDependencies": { - "grunt-contrib-jshint": "~0.6.4", - "grunt-contrib-clean": "~0.5.0", - "grunt-contrib-nodeunit": "~0.1.2", "grunt": "~0.4.1", - "grunt-simple-mocha": "~0.4.0", - "chai": "~1.8.0", - "matchdep": "~0.1.2" + "grunt-contrib-jshint": "~0.8.0", + "grunt-contrib-clean": "~0.5.0", + "matchdep": "~0.1.2", + "grunt-vows": "~0.4.0", + "vows": "~0.7.0", + "grunt-contrib-watch": "~0.5.0" }, "peerDependencies": { "grunt": "~0.4.1" diff --git a/tasks/deployments.js b/tasks/deployments.js index f487b90..1d377ac 100644 --- a/tasks/deployments.js +++ b/tasks/deployments.js @@ -178,12 +178,13 @@ module.exports = function(grunt) { function db_dump(config, output_paths) { var cmd; + var ignoreTables; grunt.file.mkdir(output_paths.dir); // 1) Get array of tables to ignore, as defined in the config, and format it correctly if( config.ignoreTables ) { - var ignoreTables = '--ignore-table=' + config.database + "." + config.ignoreTables.join(' --ignore-table='+config.database+'.'); + ignoreTables = '--ignore-table=' + config.database + "." + config.ignoreTables.join(' --ignore-table='+config.database+'.'); } // 2) Compile MYSQL cmd via Lo-Dash template string diff --git a/test/deployments_test.js b/test/deployments_test.js index 08e8065..8d24726 100644 --- a/test/deployments_test.js +++ b/test/deployments_test.js @@ -1,7 +1,3 @@ -'use strict'; - - - var chai = require("chai"); var expect = require('chai').expect; From 3286b4818b979e803808bac921b8651f0e289536 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 2 Jan 2014 12:26:45 +0000 Subject: [PATCH 08/28] Create initial test files. Port dbReplace task to library directory. --- Gruntfile.js | 10 ++++++++-- lib/dbReplace.js | 22 ++++++++++++++++++++++ tasks/deployments.js | 29 +++++++++-------------------- test/deployments_test.js | 23 ----------------------- test/options_test.js | 20 ++++++++++++++++++++ test/package_test.js | 29 +++++++++++++++++++++++++++++ 6 files changed, 88 insertions(+), 45 deletions(-) create mode 100644 lib/dbReplace.js create mode 100644 test/options_test.js create mode 100644 test/package_test.js diff --git a/Gruntfile.js b/Gruntfile.js index 8bea82d..153be13 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -6,6 +6,8 @@ * Licensed under the MIT license. */ +'use strict'; + module.exports = function(grunt) { // load all grunt tasks @@ -81,8 +83,12 @@ module.exports = function(grunt) { }, deployments: { - - } + options: { + backups_dir: '' + }, + local: '<%= db_fixture.local %>', // make sure you've created valid fixture DB creds + develop: '<%= db_fixture.develop %>' // make sure you've created valid fixture DB creds + }, }); diff --git a/lib/dbReplace.js b/lib/dbReplace.js new file mode 100644 index 0000000..02cb784 --- /dev/null +++ b/lib/dbReplace.js @@ -0,0 +1,22 @@ +'use strict'; + +var grunt = require('grunt'); +var shell = require('shelljs'); + +function dbReplace(search,replace,output_file) { + + var cmd = grunt.template.process( "sed -i '' 's#<%= search %>#<%= replace %>#g' <%= path %>", { + data: { + search: search, + replace: replace, + path: output_file + } + }); + + grunt.log.writeln("Replacing '" + search + "' with '" + replace + "' in the database."); + // Execute cmd + shell.exec(cmd); + grunt.log.oklns("Database references succesfully updated."); +} + +module.exports = dbReplace; \ No newline at end of file diff --git a/tasks/deployments.js b/tasks/deployments.js index 1d377ac..d553526 100644 --- a/tasks/deployments.js +++ b/tasks/deployments.js @@ -8,12 +8,15 @@ 'use strict'; +// Global var shell = require('shelljs'); -module.exports = function(grunt) { - +// Library modules +var dbReplace = require('../lib/dbReplace'); +// Only Grunt registration within this "exports" +module.exports = function(grunt) { /** * DB PUSH @@ -47,7 +50,7 @@ module.exports = function(grunt) { db_dump(local_options, local_backup_paths); // Search and Replace database refs - db_replace( local_options.url, target_options.url, local_backup_paths.file ); + dbReplace( local_options.url, target_options.url, local_backup_paths.file ); // Dump target DB db_dump(target_options, target_backup_paths); @@ -91,7 +94,7 @@ module.exports = function(grunt) { // Dump Target DB db_dump(target_options, target_backup_paths ); - db_replace(target_options.url,local_options.url,target_backup_paths.file); + dbReplace(target_options.url,local_options.url,target_backup_paths.file); // Backup Local DB db_dump(local_options, local_backup_paths); @@ -227,21 +230,7 @@ module.exports = function(grunt) { } - function db_replace(search,replace,output_file) { - - var cmd = grunt.template.process(tpls.search_replace, { - data: { - search: search, - replace: replace, - path: output_file - } - }); - - grunt.log.writeln("Replacing '" + search + "' with '" + replace + "' in the database."); - // Execute cmd - shell.exec(cmd); - grunt.log.oklns("Database references succesfully updated."); - } + @@ -255,7 +244,7 @@ module.exports = function(grunt) { backup_path: "<%= backups_dir %>/<%= env %>/<%= date %>/<%= time %>", - search_replace: "sed -i '' 's#<%= search %>#<%= replace %>#g' <%= path %>", + mysqldump: "mysqldump -h <%= host %> -u<%= user %> -p<%= pass %> -P<%= port %> <%= database %> <%= ignoreTables %>", diff --git a/test/deployments_test.js b/test/deployments_test.js index 8d24726..e69de29 100644 --- a/test/deployments_test.js +++ b/test/deployments_test.js @@ -1,23 +0,0 @@ -var chai = require("chai"); -var expect = require('chai').expect; - -var grunt = require('grunt'); -var deployments = require('../tasks/deployments.js'); - - - -describe('Check Grunt Registration', function(){ - it('registers deployments task with Grunt', function(){ - expect(deployments).to.exist; - expect(deployments).to.be.a('function'); - }); - - it('registers db_pull task with Grunt', function(){ - expect(grunt.task._tasks['db_pull']).to.exist; - }); - - it('registers db_push task with Grunt', function(){ - expect(grunt.task._tasks['db_push']).to.exist; - }); -}); - diff --git a/test/options_test.js b/test/options_test.js new file mode 100644 index 0000000..6eb0d56 --- /dev/null +++ b/test/options_test.js @@ -0,0 +1,20 @@ +/* 'use strict'; + +var vows = require("vows"), + assert = require("assert"), + mysqldumpwrapper = require('../lib/mysqldumpwrapper.js'); + + +exports.suite = vows.describe("Package options tests").addBatch({ + "The User option": { + topic: function() { + mysqldumpwrapper({ + + }); + }, + "errors if undefined or empty": function (topic) { + assert.throws(topic, Error); + } + } +}); + */ \ No newline at end of file diff --git a/test/package_test.js b/test/package_test.js new file mode 100644 index 0000000..e0c58ad --- /dev/null +++ b/test/package_test.js @@ -0,0 +1,29 @@ +'use strict'; + +var grunt = require('grunt'), + vows = require("vows"), + assert = require("assert"), + db_replace = require('../lib/db_replace'); + + + +exports.suite = vows.describe("Search and Replace").addBatch({ + "The db_replace task": { + topic: db_replace, + "is not null": function (topic) { + assert.isNotNull(topic); + }, + "is a function": function (topic) { + assert.isFunction(topic); + }, + } +}); + + + + + + + + + From 07b689c81879e466cfd8ca353af5ed15036a33da Mon Sep 17 00:00:00 2001 From: David Date: Thu, 2 Jan 2014 12:44:18 +0000 Subject: [PATCH 09/28] Create master tpls file to facilitate sharing of global template vars across tasks --- lib/dbDump.js | 61 ++++++++++++++++++++++++++++++++ lib/dbReplace.js | 3 +- lib/tpls.js | 20 +++++++++++ tasks/deployments.js | 83 ++++---------------------------------------- test/package_test.js | 11 +++--- 5 files changed, 95 insertions(+), 83 deletions(-) create mode 100644 lib/dbDump.js create mode 100644 lib/tpls.js diff --git a/lib/dbDump.js b/lib/dbDump.js new file mode 100644 index 0000000..94f74af --- /dev/null +++ b/lib/dbDump.js @@ -0,0 +1,61 @@ +'use strict'; + +var grunt = require('grunt'); +var shell = require('shelljs'); +var tpls = require('../lib/tpls'); + +/** + * Dumps a MYSQL database to a suitable backup location + */ +function dbDump(config, output_paths) { + + var cmd; + var ignoreTables; + + grunt.file.mkdir(output_paths.dir); + + // 1) Get array of tables to ignore, as defined in the config, and format it correctly + if( config.ignoreTables ) { + ignoreTables = '--ignore-table=' + config.database + "." + config.ignoreTables.join(' --ignore-table='+config.database+'.'); + } + + // 2) Compile MYSQL cmd via Lo-Dash template string + var tpl_mysqldump = grunt.template.process(tpls.mysqldump, { + data: { + user: config.user, + pass: config.pass, + database: config.database, + host: config.host, + port: config.port || 3306, + ignoreTables: ignoreTables || '' + } + }); + + // 3) Test whether MYSQL DB is local or whether requires remote access via SSH + + if (typeof config.ssh_host === "undefined") { // it's a local connection + grunt.log.writeln("Creating DUMP of local database"); + cmd = tpl_mysqldump; + + } else { // it's a remote connection + var tpl_ssh = grunt.template.process(tpls.ssh, { + data: { + host: config.ssh_host + } + }); + grunt.log.writeln("Creating DUMP of remote database"); + + cmd = tpl_ssh + " \\ " + tpl_mysqldump; + } + + // Capture output... + var output = shell.exec(cmd, {silent: true}).output; + + // Write output to file using native Grunt methods + grunt.file.write( output_paths.file, output ); + + grunt.log.oklns("Database DUMP succesfully exported to: " + output_paths.file); + +} + +module.exports = dbDump; \ No newline at end of file diff --git a/lib/dbReplace.js b/lib/dbReplace.js index 02cb784..e9f6a38 100644 --- a/lib/dbReplace.js +++ b/lib/dbReplace.js @@ -2,10 +2,11 @@ var grunt = require('grunt'); var shell = require('shelljs'); +var tpls = require('../lib/tpls'); function dbReplace(search,replace,output_file) { - var cmd = grunt.template.process( "sed -i '' 's#<%= search %>#<%= replace %>#g' <%= path %>", { + var cmd = grunt.template.process( tpls.search_replace, { data: { search: search, replace: replace, diff --git a/lib/tpls.js b/lib/tpls.js new file mode 100644 index 0000000..3466e90 --- /dev/null +++ b/lib/tpls.js @@ -0,0 +1,20 @@ +/** + * Lo-Dash Template Helpers + * http://lodash.com/docs/#template + * https://github.com/gruntjs/grunt/wiki/grunt.template + */ +var tpls = { + + search_replace: "sed -i '' 's#<%= search %>#<%= replace %>#g' <%= path %>", + + backup_path: "<%= backups_dir %>/<%= env %>/<%= date %>/<%= time %>", + + mysql: "mysql -h <%= host %> -u <%= user %> -p<%= pass %> -P<%= port %> <%= database %>", + + ssh: "ssh <%= host %>", + + mysqldump: "mysqldump -h <%= host %> -u<%= user %> -p<%= pass %> -P<%= port %> <%= database %> <%= ignoreTables %>" +}; + + +module.exports = tpls; \ No newline at end of file diff --git a/tasks/deployments.js b/tasks/deployments.js index d553526..4083bfa 100644 --- a/tasks/deployments.js +++ b/tasks/deployments.js @@ -12,8 +12,9 @@ var shell = require('shelljs'); // Library modules -var dbReplace = require('../lib/dbReplace'); - +var tpls = require('../lib/tpls'); +var dbReplace = require('../lib/dbReplace'); +var dbDump = require('../lib/dbDump'); // Only Grunt registration within this "exports" module.exports = function(grunt) { @@ -47,13 +48,13 @@ module.exports = function(grunt) { // Dump local DB - db_dump(local_options, local_backup_paths); + dbDump(local_options, local_backup_paths); // Search and Replace database refs dbReplace( local_options.url, target_options.url, local_backup_paths.file ); // Dump target DB - db_dump(target_options, target_backup_paths); + dbDump(target_options, target_backup_paths); // Import dump to target DB db_import(target_options, local_backup_paths.file); @@ -92,12 +93,12 @@ module.exports = function(grunt) { grunt.log.subhead("Pulling database from '" + target_options.title + "' into Local"); // Dump Target DB - db_dump(target_options, target_backup_paths ); + dbDump(target_options, target_backup_paths ); dbReplace(target_options.url,local_options.url,target_backup_paths.file); // Backup Local DB - db_dump(local_options, local_backup_paths); + dbDump(local_options, local_backup_paths); // Import dump into Local db_import(local_options,target_backup_paths.file); @@ -175,85 +176,15 @@ module.exports = function(grunt) { - /** - * Dumps a MYSQL database to a suitable backup location - */ - function db_dump(config, output_paths) { - - var cmd; - var ignoreTables; - - grunt.file.mkdir(output_paths.dir); - - // 1) Get array of tables to ignore, as defined in the config, and format it correctly - if( config.ignoreTables ) { - ignoreTables = '--ignore-table=' + config.database + "." + config.ignoreTables.join(' --ignore-table='+config.database+'.'); - } - - // 2) Compile MYSQL cmd via Lo-Dash template string - var tpl_mysqldump = grunt.template.process(tpls.mysqldump, { - data: { - user: config.user, - pass: config.pass, - database: config.database, - host: config.host, - port: config.port || 3306, - ignoreTables: ignoreTables || '' - } - }); - - // 3) Test whether MYSQL DB is local or whether requires remote access via SSH - - if (typeof config.ssh_host === "undefined") { // it's a local connection - grunt.log.writeln("Creating DUMP of local database"); - cmd = tpl_mysqldump; - - } else { // it's a remote connection - var tpl_ssh = grunt.template.process(tpls.ssh, { - data: { - host: config.ssh_host - } - }); - grunt.log.writeln("Creating DUMP of remote database"); - - cmd = tpl_ssh + " \\ " + tpl_mysqldump; - } - - // Capture output... - var output = shell.exec(cmd, {silent: true}).output; - - // Write output to file using native Grunt methods - grunt.file.write( output_paths.file, output ); - - grunt.log.oklns("Database DUMP succesfully exported to: " + output_paths.file); - - } - - - - - /** - * Lo-Dash Template Helpers - * http://lodash.com/docs/#template - * https://github.com/gruntjs/grunt/wiki/grunt.template - */ - var tpls = { - - backup_path: "<%= backups_dir %>/<%= env %>/<%= date %>/<%= time %>", - - mysqldump: "mysqldump -h <%= host %> -u<%= user %> -p<%= pass %> -P<%= port %> <%= database %> <%= ignoreTables %>", - mysql: "mysql -h <%= host %> -u <%= user %> -p<%= pass %> -P<%= port %> <%= database %>", - ssh: "ssh <%= host %>", - }; diff --git a/test/package_test.js b/test/package_test.js index e0c58ad..c7eb9c7 100644 --- a/test/package_test.js +++ b/test/package_test.js @@ -3,19 +3,18 @@ var grunt = require('grunt'), vows = require("vows"), assert = require("assert"), - db_replace = require('../lib/db_replace'); + dbReplace = require('../lib/dbReplace'), + dbDump = require('../lib/dbDump'); exports.suite = vows.describe("Search and Replace").addBatch({ - "The db_replace task": { - topic: db_replace, + "The dbReplace task": { + topic: dbReplace, "is not null": function (topic) { assert.isNotNull(topic); }, - "is a function": function (topic) { - assert.isFunction(topic); - }, + } }); From 669e3efd043dd679c346b735929af94a430731b9 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 2 Jan 2014 13:03:55 +0000 Subject: [PATCH 10/28] Ported all library functions to individual files within lib dir. --- lib/dbImport.js | 49 ++++++++++++++++++ lib/dbReplace.js | 3 +- lib/generateBackupPaths.js | 30 +++++++++++ tasks/deployments.js | 103 +++++-------------------------------- 4 files changed, 93 insertions(+), 92 deletions(-) create mode 100644 lib/dbImport.js create mode 100644 lib/generateBackupPaths.js diff --git a/lib/dbImport.js b/lib/dbImport.js new file mode 100644 index 0000000..c8c1eb0 --- /dev/null +++ b/lib/dbImport.js @@ -0,0 +1,49 @@ +'use strict'; + +var grunt = require('grunt'); +var shell = require('shelljs'); +var tpls = require('../lib/tpls'); + +/** + * Imports a .sql file into the DB provided + */ +function dbImport(config, src) { + + var cmd; + + // 1) Create cmd string from Lo-Dash template + var tpl_mysql = grunt.template.process(tpls.mysql, { + data: { + host: config.host, + user: config.user, + pass: config.pass, + database: config.database, + path: src, + port: config.port || 3306 + } + }); + + + // 2) Test whether target MYSQL DB is local or whether requires remote access via SSH + if (typeof config.ssh_host === "undefined") { // it's a local connection + grunt.log.writeln("Importing into local database"); + cmd = tpl_mysql + " < " + src; + } else { // it's a remote connection + var tpl_ssh = grunt.template.process(tpls.ssh, { + data: { + host: config.ssh_host + } + }); + + grunt.log.writeln("Importing DUMP into remote database"); + + cmd = tpl_ssh + " '" + tpl_mysql + "' < " + src; + } + + // Execute cmd + shell.exec(cmd); + + grunt.log.oklns("Database imported succesfully"); +} + +module.exports = dbImport; \ No newline at end of file diff --git a/lib/dbReplace.js b/lib/dbReplace.js index e9f6a38..82efa41 100644 --- a/lib/dbReplace.js +++ b/lib/dbReplace.js @@ -15,7 +15,8 @@ function dbReplace(search,replace,output_file) { }); grunt.log.writeln("Replacing '" + search + "' with '" + replace + "' in the database."); - // Execute cmd + + // Execute cmd shell.exec(cmd); grunt.log.oklns("Database references succesfully updated."); } diff --git a/lib/generateBackupPaths.js b/lib/generateBackupPaths.js new file mode 100644 index 0000000..8a95e34 --- /dev/null +++ b/lib/generateBackupPaths.js @@ -0,0 +1,30 @@ +'use strict'; + +var grunt = require('grunt'); +var shell = require('shelljs'); +var tpls = require('../lib/tpls'); + + +function generateBackupPaths(target, task_options) { + + var rtn = []; + + var backups_dir = task_options['backups_dir'] || "backups"; + + // Create suitable backup directory paths + rtn['dir'] = grunt.template.process(tpls.backup_path, { + data: { + backups_dir: backups_dir, + env: target, + date: grunt.template.today('yyyymmdd'), + time: grunt.template.today('HH-MM-ss'), + } + }); + + + rtn['file'] = rtn['dir'] + '/db_backup.sql'; + + return rtn; +} + +module.exports = generateBackupPaths; \ No newline at end of file diff --git a/tasks/deployments.js b/tasks/deployments.js index 4083bfa..7c77d13 100644 --- a/tasks/deployments.js +++ b/tasks/deployments.js @@ -12,9 +12,12 @@ var shell = require('shelljs'); // Library modules -var tpls = require('../lib/tpls'); -var dbReplace = require('../lib/dbReplace'); -var dbDump = require('../lib/dbDump'); +var tpls = require('../lib/tpls'); +var dbReplace = require('../lib/dbReplace'); +var dbDump = require('../lib/dbDump'); +var dbImport = require('../lib/dbImport'); +var generateBackupPaths = require('../lib/generateBackupPaths'); + // Only Grunt registration within this "exports" module.exports = function(grunt) { @@ -40,8 +43,8 @@ module.exports = function(grunt) { var local_options = grunt.config.get('deployments').local; // Generate required backup directories and paths - var local_backup_paths = generate_backup_paths("local", task_options); - var target_backup_paths = generate_backup_paths(target, task_options); + var local_backup_paths = generateBackupPaths("local", task_options); + var target_backup_paths = generateBackupPaths(target, task_options); grunt.log.subhead("Pushing database from 'Local' to '" + target_options.title + "'"); @@ -57,7 +60,7 @@ module.exports = function(grunt) { dbDump(target_options, target_backup_paths); // Import dump to target DB - db_import(target_options, local_backup_paths.file); + dbImport(target_options, local_backup_paths.file); grunt.log.subhead("Operations completed"); }); @@ -86,8 +89,8 @@ module.exports = function(grunt) { var local_options = grunt.config.get('deployments').local; // Generate required backup directories and paths - var local_backup_paths = generate_backup_paths("local", task_options); - var target_backup_paths = generate_backup_paths(target, task_options); + var local_backup_paths = generateBackupPaths("local", task_options); + var target_backup_paths = generateBackupPaths(target, task_options); // Start execution grunt.log.subhead("Pulling database from '" + target_options.title + "' into Local"); @@ -101,93 +104,11 @@ module.exports = function(grunt) { dbDump(local_options, local_backup_paths); // Import dump into Local - db_import(local_options,target_backup_paths.file); + dbImport(local_options,target_backup_paths.file); grunt.log.subhead("Operations completed"); }); - - - - function generate_backup_paths(target, task_options) { - - var rtn = []; - - var backups_dir = task_options['backups_dir'] || "backups"; - - // Create suitable backup directory paths - rtn['dir'] = grunt.template.process(tpls.backup_path, { - data: { - backups_dir: backups_dir, - env: target, - date: grunt.template.today('yyyymmdd'), - time: grunt.template.today('HH-MM-ss'), - } - }); - - - rtn['file'] = rtn['dir'] + '/db_backup.sql'; - - return rtn; - } - - - /** - * Imports a .sql file into the DB provided - */ - function db_import(config, src) { - - var cmd; - - // 1) Create cmd string from Lo-Dash template - var tpl_mysql = grunt.template.process(tpls.mysql, { - data: { - host: config.host, - user: config.user, - pass: config.pass, - database: config.database, - path: src, - port: config.port || 3306 - } - }); - - - // 2) Test whether target MYSQL DB is local or whether requires remote access via SSH - if (typeof config.ssh_host === "undefined") { // it's a local connection - grunt.log.writeln("Importing into local database"); - cmd = tpl_mysql + " < " + src; - } else { // it's a remote connection - var tpl_ssh = grunt.template.process(tpls.ssh, { - data: { - host: config.ssh_host - } - }); - - grunt.log.writeln("Importing DUMP into remote database"); - - cmd = tpl_ssh + " '" + tpl_mysql + "' < " + src; - } - - // Execute cmd - shell.exec(cmd); - - grunt.log.oklns("Database imported succesfully"); - } - - - - - - - - - - - - - - - }; From fb290bb2824085e23aff22879b16e6dea9422b56 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 2 Jan 2014 13:24:04 +0000 Subject: [PATCH 11/28] Implemented basic tests for Search and Replace command --- lib/dbReplace.js | 3 +++ test/package_test.js | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/dbReplace.js b/lib/dbReplace.js index 82efa41..6c88328 100644 --- a/lib/dbReplace.js +++ b/lib/dbReplace.js @@ -19,6 +19,9 @@ function dbReplace(search,replace,output_file) { // Execute cmd shell.exec(cmd); grunt.log.oklns("Database references succesfully updated."); + + // Return for reference and test suite purposes + return cmd; } module.exports = dbReplace; \ No newline at end of file diff --git a/test/package_test.js b/test/package_test.js index c7eb9c7..855c2cb 100644 --- a/test/package_test.js +++ b/test/package_test.js @@ -10,11 +10,13 @@ var grunt = require('grunt'), exports.suite = vows.describe("Search and Replace").addBatch({ "The dbReplace task": { - topic: dbReplace, + topic: dbReplace("foo","bar","test-file.txt"), "is not null": function (topic) { assert.isNotNull(topic); }, - + "command is composed correctly": function (topic) { + assert.equal(topic, "sed -i '' 's#foo#bar#g' test-file.txt"); + } } }); From 7c5f3f1411f7b6e3012e07e7c69f1f8847ea28b6 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 2 Jan 2014 14:03:27 +0000 Subject: [PATCH 12/28] Added basic DB Dump tests. Ensured backups dir is cleaned on every test run. --- Gruntfile.js | 5 +++-- lib/dbDump.js | 22 ++++++++++++++-------- lib/dbReplace.js | 7 +++++-- test/fixtures/basic_config.json | 19 +++++++++++++++++++ test/package_test.js | 23 +++++++++++++++++++++-- 5 files changed, 62 insertions(+), 14 deletions(-) create mode 100644 test/fixtures/basic_config.json diff --git a/Gruntfile.js b/Gruntfile.js index 153be13..f8de3af 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -15,7 +15,7 @@ module.exports = function(grunt) { // Project configuration. grunt.initConfig({ - db_fixture: grunt.file.readJSON('test/fixtures/test_db.json'), + db_fixture: grunt.file.readJSON('test/fixtures/basic_config.json'), vows: { all: { options: { @@ -80,6 +80,7 @@ module.exports = function(grunt) { // Before generating any new files, remove any previously-created files. clean: { tests: ['tmp'], + backups: ['./backups'], }, deployments: { @@ -101,6 +102,6 @@ module.exports = function(grunt) { grunt.registerTask('test', ['clean', 'vows']); // By default, lint and run all tests. - grunt.registerTask('default', ['jshint', 'vows']); + grunt.registerTask('default', ['jshint', 'test']); }; diff --git a/lib/dbDump.js b/lib/dbDump.js index 94f74af..90a9939 100644 --- a/lib/dbDump.js +++ b/lib/dbDump.js @@ -1,13 +1,14 @@ 'use strict'; -var grunt = require('grunt'); -var shell = require('shelljs'); -var tpls = require('../lib/tpls'); +var grunt = require('grunt'); +var shell = require('shelljs'); +var tpls = require('../lib/tpls'); + /** * Dumps a MYSQL database to a suitable backup location */ -function dbDump(config, output_paths) { +function dbDump(config, output_paths, noExec) { var cmd; var ignoreTables; @@ -48,14 +49,19 @@ function dbDump(config, output_paths) { cmd = tpl_ssh + " \\ " + tpl_mysqldump; } - // Capture output... - var output = shell.exec(cmd, {silent: true}).output; + if (!noExec) { + // Capture output... + var output = shell.exec(cmd, {silent: true}).output; - // Write output to file using native Grunt methods - grunt.file.write( output_paths.file, output ); + // Write output to file using native Grunt methods + grunt.file.write( output_paths.file, output ); + } grunt.log.oklns("Database DUMP succesfully exported to: " + output_paths.file); + // Return for reference and test suite purposes + return cmd; + } module.exports = dbDump; \ No newline at end of file diff --git a/lib/dbReplace.js b/lib/dbReplace.js index 6c88328..41b2e58 100644 --- a/lib/dbReplace.js +++ b/lib/dbReplace.js @@ -4,7 +4,7 @@ var grunt = require('grunt'); var shell = require('shelljs'); var tpls = require('../lib/tpls'); -function dbReplace(search,replace,output_file) { +function dbReplace(search,replace,output_file, noExec) { var cmd = grunt.template.process( tpls.search_replace, { data: { @@ -17,7 +17,10 @@ function dbReplace(search,replace,output_file) { grunt.log.writeln("Replacing '" + search + "' with '" + replace + "' in the database."); // Execute cmd - shell.exec(cmd); + if (!noExec) { + shell.exec(cmd); + } + grunt.log.oklns("Database references succesfully updated."); // Return for reference and test suite purposes diff --git a/test/fixtures/basic_config.json b/test/fixtures/basic_config.json new file mode 100644 index 0000000..dab56ef --- /dev/null +++ b/test/fixtures/basic_config.json @@ -0,0 +1,19 @@ +{ + "local": { + "title": "Local", + "database": "deploy_test", + "user": "root", + "pass": "pass4burfield", + "host": "localhost", + "url": "www.deploytest.dev:8888" + }, + "develop": { + "title": "Development Server", + "database": "ddeploy_dev", + "user": "ddeploy_dev", + "pass": "test4test", + "host": "localhost", + "url": "deploytest.com.burfield-dev.com", + "ssh_host": "ddeploy@134.0.18.114" + } +} diff --git a/test/package_test.js b/test/package_test.js index 855c2cb..b3669b2 100644 --- a/test/package_test.js +++ b/test/package_test.js @@ -1,16 +1,19 @@ 'use strict'; var grunt = require('grunt'), + fs = require("fs"), vows = require("vows"), assert = require("assert"), dbReplace = require('../lib/dbReplace'), - dbDump = require('../lib/dbDump'); + dbDump = require('../lib/dbDump'), + generateBackupPaths = require('../lib/generateBackupPaths'), + basic_config = grunt.file.readJSON('test/fixtures/basic_config.json'); exports.suite = vows.describe("Search and Replace").addBatch({ "The dbReplace task": { - topic: dbReplace("foo","bar","test-file.txt"), + topic: dbReplace("foo","bar","test-file.txt", true), "is not null": function (topic) { assert.isNotNull(topic); }, @@ -21,6 +24,22 @@ exports.suite = vows.describe("Search and Replace").addBatch({ }); +exports.suite = vows.describe("DB Dump").addBatch({ + "The dbDump task": { + topic: dbDump( + basic_config.local, + generateBackupPaths("local",{}) + ), + "is not null": function (topic) { + assert.isNotNull(topic); + }, + "command is composed correctly using basic data": function (topic) { + assert.equal(topic.trim(), "mysqldump -h localhost -uroot -ppass4burfield -P3306 deploy_test"); + } + } +}); + + From c92bdc94367c2419f7081ffe267543e30172d314 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 2 Jan 2014 14:20:05 +0000 Subject: [PATCH 13/28] Add test to check that DUMP creates a file --- lib/dbDump.js | 8 +++++++- test/package_test.js | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/dbDump.js b/lib/dbDump.js index 90a9939..68f1c10 100644 --- a/lib/dbDump.js +++ b/lib/dbDump.js @@ -12,6 +12,7 @@ function dbDump(config, output_paths, noExec) { var cmd; var ignoreTables; + var rtn; grunt.file.mkdir(output_paths.dir); @@ -59,8 +60,13 @@ function dbDump(config, output_paths, noExec) { grunt.log.oklns("Database DUMP succesfully exported to: " + output_paths.file); + rtn = { + cmd: cmd, + output_file: output_paths.file + }; + // Return for reference and test suite purposes - return cmd; + return rtn; } diff --git a/test/package_test.js b/test/package_test.js index b3669b2..127d188 100644 --- a/test/package_test.js +++ b/test/package_test.js @@ -34,10 +34,20 @@ exports.suite = vows.describe("DB Dump").addBatch({ assert.isNotNull(topic); }, "command is composed correctly using basic data": function (topic) { - assert.equal(topic.trim(), "mysqldump -h localhost -uroot -ppass4burfield -P3306 deploy_test"); - } + assert.equal(topic.cmd.trim(), "mysqldump -h localhost -uroot -ppass4burfield -P3306 deploy_test"); + }, + "results in a .sql file that": { + topic: function (topic) { + fs.stat(topic.output_file, this.callback); + }, + "has contents": function (err, stat) { + console.log(stat); + assert.isNull(err); + assert.isNotZero(stat.size); + } + } } -}); +}); From 04575bd051287e7e9e2f15a8d7c9b1282c2b58d8 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 2 Jan 2014 14:42:47 +0000 Subject: [PATCH 14/28] Add conditions for output messages --- lib/dbDump.js | 14 +++++++++++--- test/package_test.js | 11 ++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/dbDump.js b/lib/dbDump.js index 68f1c10..c0c54c2 100644 --- a/lib/dbDump.js +++ b/lib/dbDump.js @@ -36,7 +36,7 @@ function dbDump(config, output_paths, noExec) { // 3) Test whether MYSQL DB is local or whether requires remote access via SSH if (typeof config.ssh_host === "undefined") { // it's a local connection - grunt.log.writeln("Creating DUMP of local database"); + grunt.log.writeln("Creating dump of local database"); cmd = tpl_mysqldump; } else { // it's a remote connection @@ -45,7 +45,7 @@ function dbDump(config, output_paths, noExec) { host: config.ssh_host } }); - grunt.log.writeln("Creating DUMP of remote database"); + grunt.log.writeln("Creating dump of remote database"); cmd = tpl_ssh + " \\ " + tpl_mysqldump; } @@ -53,12 +53,20 @@ function dbDump(config, output_paths, noExec) { if (!noExec) { // Capture output... var output = shell.exec(cmd, {silent: true}).output; + // TODO: Add test here to check whether we were able to connect // Write output to file using native Grunt methods grunt.file.write( output_paths.file, output ); } - grunt.log.oklns("Database DUMP succesfully exported to: " + output_paths.file); + if ( grunt.file.exists(output_paths.file) ) { + + grunt.log.oklns("Database dump succesfully exported to: " + output_paths.file); + } else if (noExec) { + grunt.log.oklns("Running with 'noExec' option. Database dump would have otherwise been succesfully exported to: " + output_paths.file); + } else { + grunt.fail.warn("Unable to locate database dump .sql file at " + output_paths.file, 6); + } rtn = { cmd: cmd, diff --git a/test/package_test.js b/test/package_test.js index 127d188..0efc6bb 100644 --- a/test/package_test.js +++ b/test/package_test.js @@ -11,7 +11,7 @@ var grunt = require('grunt'), -exports.suite = vows.describe("Search and Replace").addBatch({ +exports.suite = vows.describe("Basic tests").addBatch({ "The dbReplace task": { topic: dbReplace("foo","bar","test-file.txt", true), "is not null": function (topic) { @@ -19,12 +19,10 @@ exports.suite = vows.describe("Search and Replace").addBatch({ }, "command is composed correctly": function (topic) { assert.equal(topic, "sed -i '' 's#foo#bar#g' test-file.txt"); - } + }, + // add test to check against a fixture MYSQL export file that a search and replace works as expected } -}); - - -exports.suite = vows.describe("DB Dump").addBatch({ +}).addBatch({ "The dbDump task": { topic: dbDump( basic_config.local, @@ -41,7 +39,6 @@ exports.suite = vows.describe("DB Dump").addBatch({ fs.stat(topic.output_file, this.callback); }, "has contents": function (err, stat) { - console.log(stat); assert.isNull(err); assert.isNotZero(stat.size); } From 7f19ea3d98f0356c9c999577b2f61e8d4fc81013 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 2 Jan 2014 15:00:49 +0000 Subject: [PATCH 15/28] Added branching and contribution instructions --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ee15fc1..56920d1 100644 --- a/README.md +++ b/README.md @@ -254,7 +254,12 @@ A string value that represents the default target for the tasks. You can easily ## Contributing -Contributions to this plugin are most welcome. This is very much a Alpha release and so if you find a problem please consider raising a pull request or creating a Issue which describes the problem you are having and proposes a solution. +Contributions to this plugin are most welcome. Pull requests are preferred but input on open Issues is also most agreeable! + +This is very much a Alpha release and so if you find a problem please consider raising a pull request or creating a Issue which describes the problem you are having and proposes a solution. + +### Branches and merge strategy +All pull requests should merged into the `develop` branch. __Please do not merge into the `master` branch__. In lieu of a formal styleguide, take care to maintain the existing coding style. Add unit tests for any new or changed functionality. Lint and test your code using [Grunt](http://gruntjs.com/). From b57a47b8476ad172d8074d79ea9b8e073db94abe Mon Sep 17 00:00:00 2001 From: David Date: Thu, 2 Jan 2014 15:40:30 +0000 Subject: [PATCH 16/28] Added rtn object for test purposes --- lib/dbImport.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/dbImport.js b/lib/dbImport.js index c8c1eb0..82558a4 100644 --- a/lib/dbImport.js +++ b/lib/dbImport.js @@ -10,6 +10,7 @@ var tpls = require('../lib/tpls'); function dbImport(config, src) { var cmd; + var rtn; // 1) Create cmd string from Lo-Dash template var tpl_mysql = grunt.template.process(tpls.mysql, { @@ -44,6 +45,14 @@ function dbImport(config, src) { shell.exec(cmd); grunt.log.oklns("Database imported succesfully"); + + + rtn = { + cmd: cmd + }; + + // Return for reference and test suite purposes + return rtn; } module.exports = dbImport; \ No newline at end of file From 7176e487ee2dd1bb9906772a7c0ac7b132d599e5 Mon Sep 17 00:00:00 2001 From: jdsimcoe Date: Thu, 2 Jan 2014 09:29:49 -0800 Subject: [PATCH 17/28] Added support for custom SSH ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Custom SSH ports are supported on remote datasources with the “ssh_port” declaration. --- lib/dbDump.js | 3 ++- lib/dbImport.js | 3 ++- lib/tpls.js | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/dbDump.js b/lib/dbDump.js index c0c54c2..f74b5dd 100644 --- a/lib/dbDump.js +++ b/lib/dbDump.js @@ -42,7 +42,8 @@ function dbDump(config, output_paths, noExec) { } else { // it's a remote connection var tpl_ssh = grunt.template.process(tpls.ssh, { data: { - host: config.ssh_host + host: config.ssh_host, + port: config.ssh_port } }); grunt.log.writeln("Creating dump of remote database"); diff --git a/lib/dbImport.js b/lib/dbImport.js index 82558a4..826a713 100644 --- a/lib/dbImport.js +++ b/lib/dbImport.js @@ -32,7 +32,8 @@ function dbImport(config, src) { } else { // it's a remote connection var tpl_ssh = grunt.template.process(tpls.ssh, { data: { - host: config.ssh_host + host: config.ssh_host, + port: config.ssh_port } }); diff --git a/lib/tpls.js b/lib/tpls.js index 3466e90..3620266 100644 --- a/lib/tpls.js +++ b/lib/tpls.js @@ -11,7 +11,7 @@ var tpls = { mysql: "mysql -h <%= host %> -u <%= user %> -p<%= pass %> -P<%= port %> <%= database %>", - ssh: "ssh <%= host %>", + ssh: "ssh -p <%= port %> <%= host %>", mysqldump: "mysqldump -h <%= host %> -u<%= user %> -p<%= pass %> -P<%= port %> <%= database %> <%= ignoreTables %>" }; From ac5882c42035fc4e494bebbabc9c849c33087199 Mon Sep 17 00:00:00 2001 From: jdsimcoe Date: Thu, 2 Jan 2014 10:58:52 -0800 Subject: [PATCH 18/28] Updates to add Default Port of 22 --- README.md | 13 ++++++++++++- lib/dbDump.js | 24 ++++++++++++++++++------ lib/dbImport.js | 23 +++++++++++++++++------ 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 56920d1..e153e1c 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ All other targets *must* contain a valid `ssh_host` parameter. "host": "development_db_host", "url": "development_db_url", "ssh_host": "ssh_user@ssh_host", + "ssh_port": "ssh_port", "ignoreTables": ["table1","table2",...] }, "stage": { @@ -125,6 +126,7 @@ All other targets *must* contain a valid `ssh_host` parameter. "host": "stage_db_host", "url": "stage_db_url", "ssh_host": "ssh_user@ssh_host", + "ssh_port": "ssh_port", "ignoreTables": ["table1","table2",...] }, "production": { @@ -135,6 +137,7 @@ All other targets *must* contain a valid `ssh_host` parameter. "host": "production_db_host", "url": "production_db_url", "ssh_host": "ssh_user@ssh_host", + "ssh_port": "ssh_port", "ignoreTables": ["table1","table2",...] } ``` @@ -169,6 +172,7 @@ grunt.initConfig({ "host": "development_db_host", "url": "development_db_url", "ssh_host": "ssh_user@ssh_host", + "ssh_port": "ssh_port", "ignoreTables": ["table1","table2",...] }, "stage": { @@ -179,6 +183,7 @@ grunt.initConfig({ "host": "stage_db_host", "url": "stage_db_url", "ssh_host": "ssh_user@ssh_host", + "ssh_port": "ssh_port", "ignoreTables": ["table1","table2",...] }, "production": { @@ -189,6 +194,7 @@ grunt.initConfig({ "host": "production_db_host", "url": "production_db_url", "ssh_host": "ssh_user@ssh_host", + "ssh_port": "ssh_port", "ignoreTables": ["table1","table2",...] } }, @@ -229,7 +235,12 @@ Description: the string to search and replace within the database before it is m #### ssh_host Type: `String` -Description: ssh connection string in the format `SSH_USER@SSH_HOST`. The task assumes you have ssh keys setup which allow you to remote into your server without requiring the input of a password. As this is an exhaustive topic we will not cover it here but you might like to start by reading [Github's own advice](https://help.github.com/articles/generating-ssh-keys). +Description: SSH connection string in the format `SSH_USER@SSH_HOST`. The task assumes you have SSH keys setup which allow you to remote into your server without requiring the input of a password. As this is an exhaustive topic we will not cover it here but you might like to start by reading [Github's own advice](https://help.github.com/articles/generating-ssh-keys). + +#### ssh_port +Type: `String` +Default value: `22` +Description: SSH port number in the format `#####`. If you leave this off for a remote connection, the plugin will default to the standard SSH port of `22`. #### ignoreTables Type: `Array of Strings` diff --git a/lib/dbDump.js b/lib/dbDump.js index f74b5dd..7ccd53d 100644 --- a/lib/dbDump.js +++ b/lib/dbDump.js @@ -40,12 +40,24 @@ function dbDump(config, output_paths, noExec) { cmd = tpl_mysqldump; } else { // it's a remote connection - var tpl_ssh = grunt.template.process(tpls.ssh, { - data: { - host: config.ssh_host, - port: config.ssh_port - } - }); + + // Test whether an SSH port is defined and pass the default port '22' if it's not defined + if (typeof config.ssh_port === "undefined") { + var tpl_ssh = grunt.template.process(tpls.ssh, { + data: { + host: config.ssh_host, + port: '22' + } + }); + } else { + var tpl_ssh = grunt.template.process(tpls.ssh, { + data: { + host: config.ssh_host, + port: config.ssh_port + } + }); + } + grunt.log.writeln("Creating dump of remote database"); cmd = tpl_ssh + " \\ " + tpl_mysqldump; diff --git a/lib/dbImport.js b/lib/dbImport.js index 826a713..dafb5de 100644 --- a/lib/dbImport.js +++ b/lib/dbImport.js @@ -30,12 +30,23 @@ function dbImport(config, src) { grunt.log.writeln("Importing into local database"); cmd = tpl_mysql + " < " + src; } else { // it's a remote connection - var tpl_ssh = grunt.template.process(tpls.ssh, { - data: { - host: config.ssh_host, - port: config.ssh_port - } - }); + + // Test whether an SSH port is defined and pass the default port '22' if it's not defined + if (typeof config.ssh_port === "undefined") { + var tpl_ssh = grunt.template.process(tpls.ssh, { + data: { + host: config.ssh_host, + port: '22' + } + }); + } else { + var tpl_ssh = grunt.template.process(tpls.ssh, { + data: { + host: config.ssh_host, + port: config.ssh_port + } + }); + } grunt.log.writeln("Importing DUMP into remote database"); From 07561a37e9350dba3663620e0f7b2bf8c519a1e1 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 3 Jan 2014 09:29:11 +0000 Subject: [PATCH 19/28] Update default Grunt task to watch. Update testing section in README --- Gruntfile.js | 2 +- README.md | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Gruntfile.js b/Gruntfile.js index f8de3af..76fcdda 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -102,6 +102,6 @@ module.exports = function(grunt) { grunt.registerTask('test', ['clean', 'vows']); // By default, lint and run all tests. - grunt.registerTask('default', ['jshint', 'test']); + grunt.registerTask('default', ['clean', 'watch']); }; diff --git a/README.md b/README.md index 56920d1..665c74f 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,17 @@ Contributions to this plugin are most welcome. Pull requests are preferred but i This is very much a Alpha release and so if you find a problem please consider raising a pull request or creating a Issue which describes the problem you are having and proposes a solution. +### Testing +This project uses [Vows](http://vowsjs.org/) for BDD testing. Run the tests via Grunt using + +```` +grunt test +```` + +New features should pass all current tests and add new tests as required. Please feel free to contribute new/improved tests. + + + ### Branches and merge strategy All pull requests should merged into the `develop` branch. __Please do not merge into the `master` branch__. From 889bdef875e59f72df8d17b03ddd2f47bd1c403b Mon Sep 17 00:00:00 2001 From: David Date: Fri, 3 Jan 2014 09:55:45 +0000 Subject: [PATCH 20/28] Modify pull task to accept (and require) src and dest. --- tasks/deployments.js | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tasks/deployments.js b/tasks/deployments.js index 7c77d13..156e7a9 100644 --- a/tasks/deployments.js +++ b/tasks/deployments.js @@ -73,38 +73,44 @@ module.exports = function(grunt) { grunt.registerTask('db_pull', 'Pull from Database', function() { // Options - var task_options = grunt.config.get('deployments')['options']; + var task_options = grunt.config.get('deployments')['options']; - // Get the target from the CLI args - var target = grunt.option('target') || task_options['target']; + // Get the source from the CLI args + var src = grunt.option('src') || task_options['src']; + if ( typeof src === "undefined" || typeof grunt.config.get('deployments')[src] === "undefined") { + grunt.fail.warn("Invalid source provided. I cannot pull a database from nowhere! Please checked your configuration and provide a valid source.", 6); + } - if ( typeof target === "undefined" || typeof grunt.config.get('deployments')[target] === "undefined") { - grunt.fail.warn("Invalid target provided. I cannot pull a database from nowhere! Please checked your configuration and provide a valid target.", 6); + // Get the destination from the CLI args + var dest = grunt.option('dest') || task_options['dest']; + if ( typeof dest === "undefined" || typeof grunt.config.get('deployments')[dest] === "undefined") { + grunt.fail.warn("Invalid destination provided. I cannot move a database to a non existent location! Please checked your configuration and provide a valid destination.", 6); } // Grab the options from the shared "deployments" config options - var target_options = grunt.config.get('deployments')[target]; - var local_options = grunt.config.get('deployments').local; + var src_options = grunt.config.get('deployments')[src]; + var dest_options = grunt.config.get('deployments')[dest]; // Generate required backup directories and paths - var local_backup_paths = generateBackupPaths("local", task_options); - var target_backup_paths = generateBackupPaths(target, task_options); + var src_backup_paths = generateBackupPaths(src, task_options); + var dest_backup_paths = generateBackupPaths(dest, task_options); // Start execution - grunt.log.subhead("Pulling database from '" + target_options.title + "' into Local"); + grunt.log.subhead("Pulling database from '" + src_options.title + "' into " + dest_options.title); - // Dump Target DB - dbDump(target_options, target_backup_paths ); + // Dump (backup) the source DB + dbDump(src_options, src_backup_paths ); - dbReplace(target_options.url,local_options.url,target_backup_paths.file); + // Performance search and replace on DUMP + dbReplace(src_options.url,dest_options.url,src_backup_paths.file); // Backup Local DB - dbDump(local_options, local_backup_paths); + dbDump(dest_options, dest_backup_paths); // Import dump into Local - dbImport(local_options,target_backup_paths.file); + dbImport(dest_options,src_backup_paths.file); grunt.log.subhead("Operations completed"); From e3dd1e81a723103201c0fac4da5e266caf37e10c Mon Sep 17 00:00:00 2001 From: David Date: Fri, 3 Jan 2014 10:04:33 +0000 Subject: [PATCH 21/28] Update import and dump functions to provide more accurate log messages --- lib/dbDump.js | 8 ++++---- lib/dbImport.js | 7 +++---- lib/dbReplace.js | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/dbDump.js b/lib/dbDump.js index c0c54c2..be5ad23 100644 --- a/lib/dbDump.js +++ b/lib/dbDump.js @@ -33,19 +33,20 @@ function dbDump(config, output_paths, noExec) { } }); - // 3) Test whether MYSQL DB is local or whether requires remote access via SSH + // 3) Test whether MYSQL DB requires remote access via SSH + grunt.log.writeln("Creating dump of " + config.title + " database"); if (typeof config.ssh_host === "undefined") { // it's a local connection - grunt.log.writeln("Creating dump of local database"); + cmd = tpl_mysqldump; } else { // it's a remote connection + var tpl_ssh = grunt.template.process(tpls.ssh, { data: { host: config.ssh_host } }); - grunt.log.writeln("Creating dump of remote database"); cmd = tpl_ssh + " \\ " + tpl_mysqldump; } @@ -60,7 +61,6 @@ function dbDump(config, output_paths, noExec) { } if ( grunt.file.exists(output_paths.file) ) { - grunt.log.oklns("Database dump succesfully exported to: " + output_paths.file); } else if (noExec) { grunt.log.oklns("Running with 'noExec' option. Database dump would have otherwise been succesfully exported to: " + output_paths.file); diff --git a/lib/dbImport.js b/lib/dbImport.js index 82558a4..549c957 100644 --- a/lib/dbImport.js +++ b/lib/dbImport.js @@ -25,9 +25,10 @@ function dbImport(config, src) { }); - // 2) Test whether target MYSQL DB is local or whether requires remote access via SSH + // 2) Test whether target MYSQL DB requires remote access via SSH + grunt.log.writeln("Importing into " + config.title + " database"); + if (typeof config.ssh_host === "undefined") { // it's a local connection - grunt.log.writeln("Importing into local database"); cmd = tpl_mysql + " < " + src; } else { // it's a remote connection var tpl_ssh = grunt.template.process(tpls.ssh, { @@ -36,8 +37,6 @@ function dbImport(config, src) { } }); - grunt.log.writeln("Importing DUMP into remote database"); - cmd = tpl_ssh + " '" + tpl_mysql + "' < " + src; } diff --git a/lib/dbReplace.js b/lib/dbReplace.js index 41b2e58..41a8ecb 100644 --- a/lib/dbReplace.js +++ b/lib/dbReplace.js @@ -14,7 +14,7 @@ function dbReplace(search,replace,output_file, noExec) { } }); - grunt.log.writeln("Replacing '" + search + "' with '" + replace + "' in the database."); + grunt.log.writeln("Replacing '" + search + "' with '" + replace + "' in the export (.sql) file."); // Execute cmd if (!noExec) { From 6d49e93078345d19b8692d792fc5718a94e40ffa Mon Sep 17 00:00:00 2001 From: David Date: Fri, 3 Jan 2014 10:12:56 +0000 Subject: [PATCH 22/28] Modify error messages and conditionals to ensure valid src and dest (defaulting to Local for backwards compatibility). --- tasks/deployments.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tasks/deployments.js b/tasks/deployments.js index 156e7a9..6a13f57 100644 --- a/tasks/deployments.js +++ b/tasks/deployments.js @@ -77,17 +77,24 @@ module.exports = function(grunt) { // Get the source from the CLI args var src = grunt.option('src') || task_options['src']; - if ( typeof src === "undefined" || typeof grunt.config.get('deployments')[src] === "undefined") { - grunt.fail.warn("Invalid source provided. I cannot pull a database from nowhere! Please checked your configuration and provide a valid source.", 6); + if ( typeof src === "undefined") { + grunt.fail.warn("Invalid source provided. I cannot pull a database from nowhere! Please checked your CLI 'dest' argument is valid.", 6); + } else if (typeof grunt.config.get('deployments')[src] === "undefined") { + grunt.fail.warn("Invalid source provided. I cannot pull a database from nowhere! Please checked your Grunt task configuration.", 6); } // Get the destination from the CLI args var dest = grunt.option('dest') || task_options['dest']; - if ( typeof dest === "undefined" || typeof grunt.config.get('deployments')[dest] === "undefined") { - grunt.fail.warn("Invalid destination provided. I cannot move a database to a non existent location! Please checked your configuration and provide a valid destination.", 6); - } + // Default to "local" if no destination is provided + if ( typeof dest === "undefined" ) { + dest = "local"; + } + // Check destination is valid + if ( typeof grunt.config.get('deployments')[dest] === "undefined") { + grunt.fail.warn("Invalid destination provided. I cannot move a database to a non existent location! Please checked your Grunt task configuration and provide a valid destination.", 6); + } // Grab the options from the shared "deployments" config options var src_options = grunt.config.get('deployments')[src]; From 95615148a32868bd77ac1b9b40842ab7950ab57e Mon Sep 17 00:00:00 2001 From: David Date: Fri, 3 Jan 2014 10:15:46 +0000 Subject: [PATCH 23/28] Make ssh user and host separate config options --- lib/dbDump.js | 1 + lib/dbImport.js | 1 + lib/tpls.js | 2 +- test/fixtures/basic_config.json | 5 +++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/dbDump.js b/lib/dbDump.js index be5ad23..3a567a0 100644 --- a/lib/dbDump.js +++ b/lib/dbDump.js @@ -44,6 +44,7 @@ function dbDump(config, output_paths, noExec) { var tpl_ssh = grunt.template.process(tpls.ssh, { data: { + user: config.ssh_user, host: config.ssh_host } }); diff --git a/lib/dbImport.js b/lib/dbImport.js index 549c957..e228272 100644 --- a/lib/dbImport.js +++ b/lib/dbImport.js @@ -33,6 +33,7 @@ function dbImport(config, src) { } else { // it's a remote connection var tpl_ssh = grunt.template.process(tpls.ssh, { data: { + user: config.ssh_user, host: config.ssh_host } }); diff --git a/lib/tpls.js b/lib/tpls.js index 3466e90..585be7d 100644 --- a/lib/tpls.js +++ b/lib/tpls.js @@ -11,7 +11,7 @@ var tpls = { mysql: "mysql -h <%= host %> -u <%= user %> -p<%= pass %> -P<%= port %> <%= database %>", - ssh: "ssh <%= host %>", + ssh: "ssh <%= user %>@<%= host %>", mysqldump: "mysqldump -h <%= host %> -u<%= user %> -p<%= pass %> -P<%= port %> <%= database %> <%= ignoreTables %>" }; diff --git a/test/fixtures/basic_config.json b/test/fixtures/basic_config.json index dab56ef..0b9abcd 100644 --- a/test/fixtures/basic_config.json +++ b/test/fixtures/basic_config.json @@ -13,7 +13,8 @@ "user": "ddeploy_dev", "pass": "test4test", "host": "localhost", - "url": "deploytest.com.burfield-dev.com", - "ssh_host": "ddeploy@134.0.18.114" + "url": "deploytest.com.burfield-dev.com", + "ssh_user": "ddeploy", + "ssh_host": "134.0.18.114" } } From 8d348ab0b54ecc89a656fc47f9a6613d528bf482 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 3 Jan 2014 11:36:42 +0000 Subject: [PATCH 24/28] Update dbReplace to work on temporary file rather than actual backup. Preserve original DB dumps intact. Only perform search and replace on tmp file which is subsequently deleted. --- lib/dbReplace.js | 30 +++++++++++++++++++----------- lib/generateBackupPaths.js | 4 +++- package.json | 3 ++- tasks/deployments.js | 10 ++++++++-- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/lib/dbReplace.js b/lib/dbReplace.js index 41a8ecb..41d0c37 100644 --- a/lib/dbReplace.js +++ b/lib/dbReplace.js @@ -2,29 +2,37 @@ var grunt = require('grunt'); var shell = require('shelljs'); +var fs = require('fs-extra'); var tpls = require('../lib/tpls'); -function dbReplace(search,replace,output_file, noExec) { +function dbReplace(search, replace, output_path, noExec) { + // Copy DB dump .sql to temp directory + // avoids overwriting original backup + grunt.file.copy(output_path.file, output_path["file-tmp"]); + + // Perform Search and replace on the temp file (not original) var cmd = grunt.template.process( tpls.search_replace, { data: { search: search, replace: replace, - path: output_file + path: output_path["file-tmp"] } }); - grunt.log.writeln("Replacing '" + search + "' with '" + replace + "' in the export (.sql) file."); - - // Execute cmd - if (!noExec) { - shell.exec(cmd); + if ( grunt.file.exists(output_path["file-tmp"])) { + // Execute cmd + if (!noExec) { + grunt.log.writeln("Replacing '" + search + "' with '" + replace + "' in the export (.sql) path.file."); + shell.exec(cmd); + grunt.log.oklns("Database references succesfully updated."); + } + } else { + grunt.fail.warn("Search & Replace was unable to locate database dump .sql file (expected at: " + output_path.file + ").", 6); } - - grunt.log.oklns("Database references succesfully updated."); - + // Return for reference and test suite purposes - return cmd; + return cmd; } module.exports = dbReplace; \ No newline at end of file diff --git a/lib/generateBackupPaths.js b/lib/generateBackupPaths.js index 8a95e34..f0ac45e 100644 --- a/lib/generateBackupPaths.js +++ b/lib/generateBackupPaths.js @@ -22,7 +22,9 @@ function generateBackupPaths(target, task_options) { }); - rtn['file'] = rtn['dir'] + '/db_backup.sql'; + rtn['file'] = rtn['dir'] + '/db_backup.sql'; + rtn['dir-tmp'] = './tmp'; + rtn['file-tmp'] = rtn['dir-tmp'] + '/db_backup.sql'; return rtn; } diff --git a/package.json b/package.json index b434083..850c71c 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,8 @@ "test": "grunt test" }, "dependencies": { - "shelljs": "~0.2.6" + "shelljs": "~0.2.6", + "fs-extra": "~0.8.1" }, "devDependencies": { "grunt": "~0.4.1", diff --git a/tasks/deployments.js b/tasks/deployments.js index 6a13f57..eb7bb70 100644 --- a/tasks/deployments.js +++ b/tasks/deployments.js @@ -10,6 +10,7 @@ // Global var shell = require('shelljs'); +var fs = require('fs-extra'); // Library modules var tpls = require('../lib/tpls'); @@ -111,13 +112,18 @@ module.exports = function(grunt) { dbDump(src_options, src_backup_paths ); // Performance search and replace on DUMP - dbReplace(src_options.url,dest_options.url,src_backup_paths.file); + dbReplace(src_options.url,dest_options.url,src_backup_paths); // Backup Local DB dbDump(dest_options, dest_backup_paths); // Import dump into Local - dbImport(dest_options,src_backup_paths.file); + dbImport(dest_options,src_backup_paths["file-tmp"]); + + // Clean up tmp directory + fs.removeSync(src_backup_paths["dir-tmp"], function (err) { + if (err) throw err; + }); grunt.log.subhead("Operations completed"); From 0d363034d190597d00e09464369d1a6bd45a2bab Mon Sep 17 00:00:00 2001 From: David Date: Fri, 3 Jan 2014 12:17:17 +0000 Subject: [PATCH 25/28] Major updates to the README. Minor code comment updates --- README.md | 95 ++++++++++++++++++++++++++----------------------- lib/dbDump.js | 4 +-- lib/dbImport.js | 4 +-- 3 files changed, 55 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 665c74f..5cf07e2 100644 --- a/README.md +++ b/README.md @@ -43,50 +43,34 @@ grunt.initConfig({ }); ``` -**IMPORTANT NOTE:** The task is opinionated in that it assumes you are working on a local machine and pushing/pulling databases from/to that location. Thus it is imperative that you define a `local` target as part of your configuration. +**UPDATE:** The task was originally opinionated in that it once assumed you were working on a local machine and pushing/pulling databases from/to that location. Therefore it *was* imperative that you defined a `local` target as part of your configuration. The task has now been updated to allow you to avoid having to utilise a "local" target. We still advise however, that you define a "local" target as a fallback. +Please ensure you read the full Configuration documentation below before proceeding. -### Available Tasks - -The Plugin makes two new tasks available via Grunt. These are `db_pull` and `db_push`. The interface for both commands is identical: - -````grunt db_pull --target="%%TARGET%%" // replace %%TARGET%% with the target you've defined in your config ```` - -There is a single argument `--target` that is required each time you run either command. - -#### Task: db_push -The `db_push` command moves your **local** database to a **remote** database location. The following process is observed: - -1. Takes a dump of your local database -2. Runs a search and replace on the local dump file -3. Backups up the target database (remote) -4. Imports the local dump into the target database - -The `target` argument represents the remote target to which you wish to push your database. +### Available Tasks -````grunt db_push --target="develop"```` +The Plugin makes use of a **single** task `db_pull`. The interface for this command is as follows: -#### Task: db_pull +````grunt db_pull```` -The `db_pull` command pulls a **remote** database into your **local** environment. The following process is observed: +There are two flags that should be used each time you run either command: -1. Takes a dump of the remote database -2. Runs a search and replace on the dump file -3. Backups up your local database -4. Imports the remote dump into your local database +* `--src` - the source database from which you wish to export your SQL +* `--dest` - the destination database into which you would like to import your SQL -The `target` argument represents the remote target whose database you wish to pull into your local environment. Eg: +#### Example -````grunt db_pull --target="stage"```` +````grunt db_pull --src="%%SOURCE_DB%%" --dest="%%DESTINATION_DB%%"```` +__Note:__ only `src` is required. If `dest` is not provided the task will automatically assume you wish to use the `local` target defined in your Grunt task configuration. This is for ease of use and also to maintain backwards compatibility with the older CLI. -### Usage +### Configuration #### Local Target (required) -As above, the Plugin task is opinionated. It *expects* that you are working locally and pushing/pulling from/to that location. +Whilst the Plugin task (no longer) forces you to define a "local" target, we still advise that you always define one. This is because the `local` target will be used as the default destination if one is not explicity provided. -As a result, it is essential that you define a *single* target *without* an `ssh_host` parameter. This is typically named "local" for convenience. +Your local target should not require a `ssh_host` parameter and, to avoid complication, should be named exactly as `"local"`. ```js "local": { @@ -101,10 +85,10 @@ As a result, it is essential that you define a *single* target *without* an `ssh }, ``` -The task will assume that this target is equivilant to your `local` environment. You can call it anything you wish but it ***must not*** have an `ssh_host` parameter. +Again, the "local" target ***must not*** have an `ssh_host` parameter. #### Other Environment Targets -All other targets *must* contain a valid `ssh_host` parameter. +All other targets may contain valid ssh credentials. ```js "develop": { @@ -114,7 +98,8 @@ All other targets *must* contain a valid `ssh_host` parameter. "pass": "development_db_password", "host": "development_db_host", "url": "development_db_url", - "ssh_host": "ssh_user@ssh_host", + "ssh_user": "ssh_user", // UPDATE: user/host now defined separately + "ssh_host": "ssh_host", // UPDATE: user/host now defined separately "ignoreTables": ["table1","table2",...] }, "stage": { @@ -124,7 +109,8 @@ All other targets *must* contain a valid `ssh_host` parameter. "pass": "stage_db_password", "host": "stage_db_host", "url": "stage_db_url", - "ssh_host": "ssh_user@ssh_host", + "ssh_user": "ssh_user", // UPDATE: user/host now defined separately + "ssh_host": "ssh_host", // UPDATE: user/host now defined separately "ignoreTables": ["table1","table2",...] }, "production": { @@ -134,7 +120,8 @@ All other targets *must* contain a valid `ssh_host` parameter. "pass": "production_db_password", "host": "production_db_host", "url": "production_db_url", - "ssh_host": "ssh_user@ssh_host", + "ssh_user": "ssh_user", // UPDATE: user/host now defined separately + "ssh_host": "ssh_host", // UPDATE: user/host now defined separately "ignoreTables": ["table1","table2",...] } ``` @@ -227,13 +214,17 @@ Description: the port that MySQL is running on. Defaults to `3306` Type: `String` Description: the string to search and replace within the database before it is moved to the target location. Typically this is designed for use with systems such as WordPress where the `siteurl` value is [stored in the database](http://codex.wordpress.org/Changing_The_Site_URL) and is required to be updated upon migration to a new environment. It is however suitable for replacing any single value within the database before it is moved. +#### ssh_user +Type: `String` +Description: any valid ssh user. The task assumes you have ssh keys setup which allow you to remote into your server without requiring the input of a password. As this is an exhaustive topic we will not cover it here but you might like to start by reading [Github's own advice](https://help.github.com/articles/generating-ssh-keys). + #### ssh_host Type: `String` -Description: ssh connection string in the format `SSH_USER@SSH_HOST`. The task assumes you have ssh keys setup which allow you to remote into your server without requiring the input of a password. As this is an exhaustive topic we will not cover it here but you might like to start by reading [Github's own advice](https://help.github.com/articles/generating-ssh-keys). +Description: any valid ssh host string ~~in the format `SSH_USER@SSH_HOST`~~. The task assumes you have ssh keys setup which allow you to remote into your server without requiring the input of a password. As this is an exhaustive topic we will not cover it here but you might like to start by reading [Github's own advice](https://help.github.com/articles/generating-ssh-keys). #### ignoreTables -Type: `Array of Strings` -Description: tables to ignore. They won't be in the dump — neither their structure nor their content. +Type: `Array` +Description: a list of tables to ignore in array format. Tables defined here will be ommitted from the dump. Neither their structure nor their content will be included. ### Options @@ -245,18 +236,20 @@ A string value that represents the directory path (*relative* to your Grunt file You may wish to have your backups reside outside the current working directory of your Gruntfile. In which case simply provide the relative path eg: ````../../backups````. -#### options.target +#### options.target (deprecated) -Type: `String` -Default value: `` +*The task now requires `src` and `dest` parameters. If no `dest` is provided then the `local` target will be preffered.* -A string value that represents the default target for the tasks. You can easily override it using the `--target` option +~~Type: `String`~~ +~~Default value: ``~~ + +~~A string value that represents the default target for the tasks. You can easily override it using the `--target` option~~ ## Contributing Contributions to this plugin are most welcome. Pull requests are preferred but input on open Issues is also most agreeable! -This is very much a Alpha release and so if you find a problem please consider raising a pull request or creating a Issue which describes the problem you are having and proposes a solution. +I still consider this a Beta release and so if you find a problem please help me out by raising a pull request or creating a Issue which describes the problem you are having and proposes a solution. ### Testing This project uses [Vows](http://vowsjs.org/) for BDD testing. Run the tests via Grunt using @@ -274,9 +267,23 @@ All pull requests should merged into the `develop` branch. __Please do not merge In lieu of a formal styleguide, take care to maintain the existing coding style. Add unit tests for any new or changed functionality. Lint and test your code using [Grunt](http://gruntjs.com/). +## Update Guide + +### v0.4.0 + +As of `v0.4.0` the task has received several major updates. If you have used an older version of the Plugin then it's really easy to upgrade. Please check the following: + +1) You should only utilise the `db_pull` command. +2) `--target` is no longer a valid CLI parameter. Instead please pass `--src` and `--dest` which match those defined in your Grunt config. +3) In your Grunt config, check that you have defined `ssh_user` and `ssh_host` separately. `ssh_host` is now only the actual hostname. The `ssh_user` option is provided separately to accept your SSH username (see docs), +4) You are no longer forced to utilise a "Local" target. However we still advise defining one (see docs). + +If you notice any other issues please raise and issue or submit a valid pull request. + ## Release History -* 2013-12-09 v0.3.0 Added `ignoreTables` option. +* 2014-01-03   v0.4.0 Major updates to streamline task. See "Update Guide" above. +* 2013-12-09   v0.3.0 Added `ignoreTables` option. * 2013-11-12   v0.2.0   Fix escaping issues, ability to define `target` via options, README doc fixes, pass host param to mysqldump. * 2013-06-11   v0.1.0   Minor updates to docs including addtion of Release History section. * 2013-06-11   v0.0.1   Initial Plugin release. diff --git a/lib/dbDump.js b/lib/dbDump.js index 3a567a0..6d6bb72 100644 --- a/lib/dbDump.js +++ b/lib/dbDump.js @@ -36,11 +36,11 @@ function dbDump(config, output_paths, noExec) { // 3) Test whether MYSQL DB requires remote access via SSH grunt.log.writeln("Creating dump of " + config.title + " database"); - if (typeof config.ssh_host === "undefined") { // it's a local connection + if (typeof config.ssh_host === "undefined") { // we're not using SSH cmd = tpl_mysqldump; - } else { // it's a remote connection + } else { // it's a remote SSH connection var tpl_ssh = grunt.template.process(tpls.ssh, { data: { diff --git a/lib/dbImport.js b/lib/dbImport.js index e228272..70470ae 100644 --- a/lib/dbImport.js +++ b/lib/dbImport.js @@ -28,9 +28,9 @@ function dbImport(config, src) { // 2) Test whether target MYSQL DB requires remote access via SSH grunt.log.writeln("Importing into " + config.title + " database"); - if (typeof config.ssh_host === "undefined") { // it's a local connection + if (typeof config.ssh_host === "undefined") { // we're not using SSH cmd = tpl_mysql + " < " + src; - } else { // it's a remote connection + } else { // it's a remote SSH connection var tpl_ssh = grunt.template.process(tpls.ssh, { data: { user: config.ssh_user, From 62693a2b3d6e201a86e2b24f00ec3deb5567269d Mon Sep 17 00:00:00 2001 From: David Date: Fri, 3 Jan 2014 12:32:02 +0000 Subject: [PATCH 26/28] Add a section on Security to README. Explain how to keep creds out of VCS. --- README.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5cf07e2..94f50e0 100644 --- a/README.md +++ b/README.md @@ -67,8 +67,8 @@ __Note:__ only `src` is required. If `dest` is not provided the task will automa ### Configuration -#### Local Target (required) -Whilst the Plugin task (no longer) forces you to define a "local" target, we still advise that you always define one. This is because the `local` target will be used as the default destination if one is not explicity provided. +#### Local Target (recommended ~~required~~) +Whilst the Plugin task (no longer) forces you to define a `local` target, we still advise that you always define one. This is because the `local` target will be used as the default destination if one is not explicity provided. Your local target should not require a `ssh_host` parameter and, to avoid complication, should be named exactly as `"local"`. @@ -80,7 +80,6 @@ Your local target should not require a `ssh_host` parameter and, to avoid compli "pass": "local_db_password", "host": "local_db_host", "url": "local_db_url", - "ignoreTables": ["table1","table2",...] // note that the `local` target does not have an "ssh_host" }, ``` @@ -100,7 +99,6 @@ All other targets may contain valid ssh credentials. "url": "development_db_url", "ssh_user": "ssh_user", // UPDATE: user/host now defined separately "ssh_host": "ssh_host", // UPDATE: user/host now defined separately - "ignoreTables": ["table1","table2",...] }, "stage": { "title": "Stage", @@ -111,7 +109,6 @@ All other targets may contain valid ssh credentials. "url": "stage_db_url", "ssh_user": "ssh_user", // UPDATE: user/host now defined separately "ssh_host": "ssh_host", // UPDATE: user/host now defined separately - "ignoreTables": ["table1","table2",...] }, "production": { "title": "Production", @@ -122,7 +119,6 @@ All other targets may contain valid ssh credentials. "url": "production_db_url", "ssh_user": "ssh_user", // UPDATE: user/host now defined separately "ssh_host": "ssh_host", // UPDATE: user/host now defined separately - "ignoreTables": ["table1","table2",...] } ``` @@ -245,6 +241,59 @@ You may wish to have your backups reside outside the current working directory o ~~A string value that represents the default target for the tasks. You can easily override it using the `--target` option~~ +## Security + +For obvious reasons you may wish to keep your SSH and DB creds outside of source control. A simple way to achieve this is to store your credentials in an external file which is not tracked into VCS. + +I prefer to utilise a `.json` or `.yaml` file to store your targets as [Grunt provides methods](http://gruntjs.com/api/grunt.file#grunt.file.readjson) to parse both formats. + +### Example JSON file + +```json +{ + "local": { + "title": "Local", + "database": "local_db_name", + "user": "local_db_username", + "pass": "local_db_password", + "host": "local_db_host", + "url": "local_db_url", + "ignoreTables": ["table1","table2",...] + // note that the `local` target does not have an "ssh_host" + }, + "develop": { + "title": "Development", + "database": "development_db_name", + "user": "development_db_username", + "pass": "development_db_password", + "host": "development_db_host", + "url": "development_db_url", + "ssh_host": "ssh_user@ssh_host", + "ignoreTables": ["table1","table2",...] + }, +} +``` + +### Usage in Grunt config +These can then be read into your Grunt config and utilised as required. + +```js +grunt.initConfig({ + // Read external file into Grunt and parse as JSON + untracked_targets: grunt.file.readJSON('untracked_targets.json'), + + deployments: { + options: { + // options here + }, + local: '<%= untracked_targets.local %>', // reuse externally defined creds + develop: '<%= untracked_targets.develop %>' // reuse externally defined creds + }, +}); +``` + +Be sure to exclude your `.json` file from your version control system of choice. + ## Contributing Contributions to this plugin are most welcome. Pull requests are preferred but input on open Issues is also most agreeable! From b7ba97ceb4e4f5f5d42bfdc0bbf95663499fc8b3 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 3 Jan 2014 12:36:12 +0000 Subject: [PATCH 27/28] Minor amends to correct doc examples --- README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 94f50e0..3da2a28 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,8 @@ grunt.initConfig({ "pass": "development_db_password", "host": "development_db_host", "url": "development_db_url", - "ssh_host": "ssh_user@ssh_host", + "ssh_user": "ssh_user", // UPDATE: user/host now defined separately + "ssh_host": "ssh_host", // UPDATE: user/host now defined separately "ignoreTables": ["table1","table2",...] }, "stage": { @@ -161,7 +162,8 @@ grunt.initConfig({ "pass": "stage_db_password", "host": "stage_db_host", "url": "stage_db_url", - "ssh_host": "ssh_user@ssh_host", + "ssh_user": "ssh_user", // UPDATE: user/host now defined separately + "ssh_host": "ssh_host", // UPDATE: user/host now defined separately "ignoreTables": ["table1","table2",...] }, "production": { @@ -171,7 +173,8 @@ grunt.initConfig({ "pass": "production_db_password", "host": "production_db_host", "url": "production_db_url", - "ssh_host": "ssh_user@ssh_host", + "ssh_user": "ssh_user", // UPDATE: user/host now defined separately + "ssh_host": "ssh_host", // UPDATE: user/host now defined separately "ignoreTables": ["table1","table2",...] } }, @@ -258,8 +261,6 @@ I prefer to utilise a `.json` or `.yaml` file to store your targets as [Grunt pr "pass": "local_db_password", "host": "local_db_host", "url": "local_db_url", - "ignoreTables": ["table1","table2",...] - // note that the `local` target does not have an "ssh_host" }, "develop": { "title": "Development", @@ -268,8 +269,6 @@ I prefer to utilise a `.json` or `.yaml` file to store your targets as [Grunt pr "pass": "development_db_password", "host": "development_db_host", "url": "development_db_url", - "ssh_host": "ssh_user@ssh_host", - "ignoreTables": ["table1","table2",...] }, } ``` From 7600b37e3a975bc199ad0c6a038d305fc24be6a1 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 3 Jan 2014 13:02:12 +0000 Subject: [PATCH 28/28] Fix JSHint error --- tasks/deployments.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tasks/deployments.js b/tasks/deployments.js index eb7bb70..74cadb9 100644 --- a/tasks/deployments.js +++ b/tasks/deployments.js @@ -122,7 +122,9 @@ module.exports = function(grunt) { // Clean up tmp directory fs.removeSync(src_backup_paths["dir-tmp"], function (err) { - if (err) throw err; + if (err) { + throw err; + } }); grunt.log.subhead("Operations completed");