Adding replication using the "_replicator" database

Starting with 1.2.0 CouchDB added a new system database called "_replicator" to handle replications jobs. Replications are now created as entries on that database and the server will schedule and perform the replication accordingly. Entries in the "_replicator" db will be updated. This means that replication now is a completely asynchronous job that is not guaranteed to run right after the replication was started. This commit adds three new object with three object to handle this new type of replication: - replication.enable: To enable the replication of a database. - replication.query: To query the status of a replication job. - replication.disable: To disable the replication of a database. More information on this type of replication can be found: - https://wiki.apache.org/couchdb/Replication#from_1.2.0_onward - http://guide.couchdb.org/draft/replication.html - https://gist.github.com/fdmanana/832610
apache · Feb 8, 2017 · 3b18a3d · 3b18a3d
1 parent a10f6e6
commit 3b18a3d
Show file tree

Hide file tree

Showing 5 changed files with 346 additions and 3 deletions.
diff --git a/lib/nano.js b/lib/nano.js
@@ -25,6 +25,7 @@ var nano;
 
 module.exports = exports = nano = function dbScope(cfg) {
   var serverScope = {};
+  var replications = {};
 
   if (typeof cfg === 'string') {
     cfg = {url: cfg};
@@ -354,13 +355,44 @@ module.exports = exports = nano = function dbScope(cfg) {
       callback = opts;
       opts = {};
     }
-
+    // _replicate
     opts.source = _serializeAsUrl(source);
     opts.target = _serializeAsUrl(target);
 
     return relax({db: '_replicate', body: opts, method: 'POST'}, callback);
   }
 
+  // http://guide.couchdb.org/draft/replication.html
+  function enableReplication(source, target, opts, callback) {
+    if (typeof opts === 'function') {
+      callback = opts;
+      opts = {};
+    }
+    // _replicator
+    opts.source = _serializeAsUrl(source);
+    opts.target = _serializeAsUrl(target);
+
+    return relax({db: '_replicator', body: opts, method: 'POST'}, callback);
+  }
+
+  // http://guide.couchdb.org/draft/replication.html
+  function queryReplication(id, opts, callback) {
+    if (typeof opts === 'function') {
+      callback = opts;
+      opts = {};
+    }
+    return relax({db: '_replicator', method: 'GET', path: id}, callback);
+  }
+
+  // http://guide.couchdb.org/draft/replication.html
+  function disableReplication(id, rev, opts, callback) {
+    if (typeof opts === 'function') {
+      callback = opts;
+      opts = {};
+    }
+    return relax({db: '_replicator', method: 'DELETE', path: id, qs: {rev: rev}}, callback);
+  }
+
   function docModule(dbName) {
     var docScope = {};
     dbName = decodeURIComponent(dbName);
@@ -740,7 +772,18 @@ module.exports = exports = nano = function dbScope(cfg) {
       search: viewSearch,
       spatial: viewSpatial,
       view: viewDocs,
-      viewWithList: viewWithList
+      viewWithList: viewWithList,
+      replication: {
+        enable: function(target, opts, cb) {
+          return enableReplication(dbName, target, opts, cb);
+        },
+        disable: function(id, revision, opts, cb) {
+          return disableReplication(id, revision, opts, cb);
+        },
+        query: function(id, opts, cb) {
+          return queryReplication(id, opts, cb);
+        }
+      }
     };
 
     docScope.view.compact = function(ddoc, cb) {
@@ -761,6 +804,11 @@ module.exports = exports = nano = function dbScope(cfg) {
       scope: docModule,
       compact: compactDb,
       replicate: replicateDb,
+      replication: {
+        enable: enableReplication,
+        disable: disableReplication,
+        query: queryReplication
+      },
       changes: changesDb,
       follow: followDb,
       followUpdates: followUpdates,

diff --git a/tests/fixtures/database/replicator.json b/tests/fixtures/database/replicator.json
@@ -0,0 +1,117 @@
+[
+  { "method"   : "put"
+  , "path"     : "/database_replicator"
+  , "status"   : 201
+  , "response" : "{ \"ok\": true }"
+  }
+, { "method"   : "put"
+  , "path"     : "/database_replica"
+  , "status"   : 201
+  , "response" : "{ \"ok\": true }"
+  }
+, { "method"   : "put"
+  , "path"     : "/database_replica2"
+  , "status"   : 201
+  , "response" : "{ \"ok\": true }"
+  }
+, { "method"   : "put"
+  , "path"     : "/database_replica3"
+  , "status"   : 201
+  , "response" : "{ \"ok\": true }"
+  }
+, { "method"   : "put"
+  , "status"   : 201
+  , "path"     : "/database_replicator/foobar"
+  , "body"     : "{\"foo\":\"bar\"}"
+  , "response" : "{\"ok\":true,\"id\":\"foobar\",\"rev\":\"1-4c6114\"}"
+  }
+, { "method"   : "put"
+  , "status"   : 201
+  , "path"     : "/database_replicator/foobaz"
+  , "body"     : "{\"foo\":\"baz\"}"
+  , "response" : "{\"ok\":true,\"id\":\"foobaz\",\"rev\":\"1-611488\"}"
+  }
+, { "method"   : "put"
+  , "status"   : 201
+  , "path"     : "/database_replicator/barfoo"
+  , "body"     : "{\"bar\":\"foo\"}"
+  , "response" : "{\"ok\":true,\"id\":\"barfoo\",\"rev\":\"1-3cde10\"}"
+  }
+, { "method"   : "post"
+  , "status"   : 201
+  , "path"     : "/_replicator"
+  , "body"     : "{\"source\":\"database_replicator\",\"target\":\"database_replica\"}"
+  , "response" : "{\"ok\":true, \"id\": \"632c186d2c10497410f8b46ef300016e\"}"
+  }
+, { "path"     : "/_replicator/632c186d2c10497410f8b46ef300016e"
+  , "status"   : 200
+  , "response" : "{ \"_id\": \"632c186d2c10497410f8b46ef300016e\", \"_rev\": \"3-c83884542204db29b34cd9ed9e5364e1\", \"source\": \"database_replicator\", \"target\": \"database_replica\", \"owner\": null, \"_replication_state\": \"completed\", \"_replication_state_time\": \"2017-02-07T11:42:25+01:00\", \"_replication_id\": \"c1ed194ee95788f1fcade8cf5489bce9\", \"_replication_stats\": { \"revisions_checked\": 3, \"missing_revisions_found\": 3, \"docs_read\": 3, \"docs_written\": 3, \"doc_write_failures\": 0, \"checkpointed_source_seq\": 3 } }"
+  }
+, { "path"     : "/database_replica/_all_docs"
+  , "status"   : 200
+  , "response" : "{\"total_rows\":3,\"offset\":0,\"rows\":[{\"id\":\"barfoo\",\"key\":\"barfoo\",\"value\":{\"rev\":\"1-41412c293dade3fe73279cba8b4cece4\"}},{\"id\":\"foobar\",\"key\":\"foobar\",\"value\":{\"rev\":\"1-4c6114c65e295552ab1019e2b046b10e\"}},{\"id\":\"foobaz\",\"key\":\"foobaz\",\"value\":{\"rev\":\"1-cfa20dddac397da5bf0be2b50fb472fe\"}}]}"
+  }
+, { "method"   : "delete"
+  , "status"   : 200
+  , "path"     : "/_replicator/632c186d2c10497410f8b46ef300016e?rev=3-c83884542204db29b34cd9ed9e5364e1"
+  , "response" : "{\"ok\":true, \"id\": \"632c186d2c10497410f8b46ef300016e\"}"
+  }
+, { "method"   : "post"
+  , "status"   : 201
+  , "path"     : "/_replicator"
+  , "body"     : "{\"source\":\"http://localhost:5984/database_replicator\",\"target\":\"database_replica2\"}"
+  , "response" : "{\"ok\":true, \"id\": \"632c186d2c10497410f8b46ef300018f\"}"
+  }
+, { "path"     : "/_replicator/632c186d2c10497410f8b46ef300018f"
+  , "status"   : 200
+  , "response" : "{ \"_id\": \"632c186d2c10497410f8b46ef300018f\", \"_rev\": \"3-c83884542204db29b34cd9ed9e5364e1\", \"source\": \"database_replicator\", \"target\": \"database_replica2\", \"owner\": null, \"_replication_state\": \"completed\", \"_replication_state_time\": \"2017-02-07T11:42:25+01:00\", \"_replication_id\": \"c1ed194ee95788f1fcade8cf5489bce9\", \"_replication_stats\": { \"revisions_checked\": 3, \"missing_revisions_found\": 3, \"docs_read\": 3, \"docs_written\": 3, \"doc_write_failures\": 0, \"checkpointed_source_seq\": 3 } }"
+  }
+, { "path"     : "/database_replica2/_all_docs"
+  , "status"   : 200
+  , "response" : "{\"total_rows\":3,\"offset\":0,\"rows\":[{\"id\":\"barfoo\",\"key\":\"barfoo\",\"value\":{\"rev\":\"1-41412c293dade3fe73279cba8b4cece4\"}},{\"id\":\"foobar\",\"key\":\"foobar\",\"value\":{\"rev\":\"1-4c6114c65e295552ab1019e2b046b10e\"}},{\"id\":\"foobaz\",\"key\":\"foobaz\",\"value\":{\"rev\":\"1-cfa20dddac397da5bf0be2b50fb472fe\"}}]}"
+  }
+, { "method"   : "delete"
+  , "status"   : 200
+  , "path"     : "/_replicator/632c186d2c10497410f8b46ef300018f?rev=3-c83884542204db29b34cd9ed9e5364e1"
+  , "response" : "{\"ok\":true, \"id\": \"632c186d2c10497410f8b46ef300018f\"}"
+  }
+, { "method"   : "post"
+  , "status"   : 201
+  , "path"     : "/_replicator"
+  , "body"     : "{\"source\":\"database_replicator\",\"target\":\"database_replica3\"}"
+  , "response" : "{\"ok\":true, \"id\": \"632c186d2c10497410f8b46ef3000200\"}"
+  }
+, { "path"     : "/_replicator/632c186d2c10497410f8b46ef3000200"
+  , "status"   : 200
+  , "response" : "{ \"_id\": \"632c186d2c10497410f8b46ef3000200\", \"_rev\": \"3-c83884542204db29b34cd9ed9e5364e1\", \"source\": \"database_replicator\", \"target\": \"database_replica3\", \"owner\": null, \"_replication_state\": \"completed\", \"_replication_state_time\": \"2017-02-07T11:42:25+01:00\", \"_replication_id\": \"c1ed194ee95788f1fcade8cf5489bce9\", \"_replication_stats\": { \"revisions_checked\": 3, \"missing_revisions_found\": 3, \"docs_read\": 3, \"docs_written\": 3, \"doc_write_failures\": 0, \"checkpointed_source_seq\": 3 } }"
+  }
+, { "path"     : "/database_replica3/_all_docs"
+  , "status"   : 200
+  , "response" : "{\"total_rows\":3,\"offset\":0,\"rows\":[{\"id\":\"barfoo\",\"key\":\"barfoo\",\"value\":{\"rev\":\"1-41412c293dade3fe73279cba8b4cece4\"}},{\"id\":\"foobar\",\"key\":\"foobar\",\"value\":{\"rev\":\"1-4c6114c65e295552ab1019e2b046b10e\"}},{\"id\":\"foobaz\",\"key\":\"foobaz\",\"value\":{\"rev\":\"1-cfa20dddac397da5bf0be2b50fb472fe\"}}]}"
+  }
+, { "method"   : "delete"
+  , "status"   : 200
+  , "path"     : "/_replicator/632c186d2c10497410f8b46ef3000200?rev=3-c83884542204db29b34cd9ed9e5364e1"
+  , "response" : "{\"ok\":true, \"id\": \"632c186d2c10497410f8b46ef3000200\"}"
+  }
+, { "method"   : "delete"
+  , "path"     : "/database_replicator"
+  , "status"   : 200
+  , "response" : "{ \"ok\": true }"
+  }
+, { "method"   : "delete"
+  , "path"     : "/database_replica"
+  , "status"   : 200
+  , "response" : "{ \"ok\": true }"
+  }
+, { "method"   : "delete"
+  , "path"     : "/database_replica2"
+  , "status"   : 200
+  , "response" : "{ \"ok\": true }"
+  }
+, { "method"   : "delete"
+  , "path"     : "/database_replica3"
+  , "status"   : 200
+  , "response" : "{ \"ok\": true }"
+  }
+]
diff --git a/tests/helpers/unit.js b/tests/helpers/unit.js
@@ -56,7 +56,18 @@ helpers.unit = function(method, error) {
       // are at the top level in nano
       //
       if(method[0] === 'database') {
-        f = cli.server.db[method[1]];
+        //
+        // Due to the way this harness is designed we cannot differentiate between different methods
+        // when those methods are embedded on an object.
+        // We have two options, either we hardcode the resolution or we write a full harness that
+        // can differentiate between methods embedded on an object.
+        // I go the hardcoded route for now.
+        //
+        if (method[1] === 'replicator') {
+          f = cli.server.db.replication.enable;
+        } else {
+          f = cli.server.db[method[1]];
+        }
       } else if(method[0] === 'view' && method[1] === 'compact') {
         f = cli.view.compact;
       } else if(!~['multipart', 'attachment'].indexOf(method[0])) {

diff --git a/tests/integration/database/replicator.js b/tests/integration/database/replicator.js
@@ -0,0 +1,129 @@
+// Licensed under the Apache License, Version 2.0 (the 'License'); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+'use strict';
+
+var async = require('async');
+var helpers = require('../../helpers/integration');
+var harness = helpers.harness(__filename);
+var it = harness.it;
+var db = harness.locals.db;
+var nano = harness.locals.nano;
+
+var replica;
+var replica2;
+var replica3;
+
+it('should insert a bunch of items', helpers.insertThree);
+
+it('creates a bunch of database replicas', function(assert) {
+    async.forEach(['database_replica', 'database_replica2', 'database_replica3'],
+    nano.db.create, function(error) {
+        assert.equal(error, undefined, 'created database(s)');
+        assert.end();
+    });
+});
+
+it('should be able to replicate (replicator) three docs', function(assert) {
+  replica = nano.use('database_replica');
+  db.replication.enable('database_replica', function(error, data) {
+    assert.equal(error, null, 'replication should not fail');
+    assert.true(data, 'replication should be scheduled');
+    assert.true(data.ok, 'replication should be scheduled');
+    assert.true(data.id, 'replication should return the id to query back');
+    function waitForReplication() {
+      setTimeout(function() {
+        db.replication.query(data.id, function(error, reply) {
+          assert.equal(reply.target, 'database_replica', 'target db should match');
+          assert.equal(reply._replication_state, 'completed', 'replication should have completed');
+          replica.list(function(error, list) {
+            assert.equal(error, null, 'should be able to invoke list');
+            assert.equal(list['total_rows'], 3, 'and have three documents');
+            db.replication.disable(reply._id, reply._rev, function(error, disabled) {
+              assert.true(disabled, 'should not be null');
+              assert.true(disabled.ok, 'should have stopped the replication');
+              assert.end();
+            });  
+          })
+        })
+      }, 
+      3000)
+    };
+    waitForReplication();
+  });
+});
+
+it('should be able to replicate (replicator) to a `nano` object', function(assert) {
+  replica2 = nano.use('database_replica2');
+  nano.db.replication.enable(db, 'database_replica2', function(error, data) {
+    assert.equal(error, null, 'replication should not fail');
+    assert.true(data, 'replication should be scheduled');
+    assert.true(data.ok, 'replication should be scheduled');
+    assert.true(data.id, 'replication should return the id to query back');
+    function waitForReplication() {
+      setTimeout(function() {
+        nano.db.replication.query(data.id, function(error, reply) {
+          assert.equal(reply.target, 'database_replica2', 'target db should match');
+          assert.equal(reply._replication_state, 'completed', 'replication should have completed');
+          replica2.list(function(error, list) {
+            assert.equal(error, null, 'should be able to invoke list');
+            assert.equal(list['total_rows'], 3, 'and have three documents');
+            nano.db.replication.disable(reply._id, reply._rev, function(error, disabled) {
+              assert.true(disabled, 'should not be null');
+              assert.true(disabled.ok, 'should have stopped the replication');
+              assert.end();
+            });
+          });
+        })
+      }, 
+      3000)
+    };
+    waitForReplication();
+  });
+});
+
+it('should be able to replicate (replicator) with params', function(assert) {
+  replica3 = nano.use('database_replica3');
+  db.replication.enable('database_replica3', {}, function(error, data) {
+    assert.equal(error, null, 'replication should not fail');
+    assert.true(data, 'replication should be scheduled');
+    assert.true(data.ok, 'replication should be scheduled');
+    assert.true(data.id, 'replication should return the id to query back');
+    function waitForReplication() {
+      setTimeout(function() {
+        db.replication.query(data.id, function(error, reply) {
+          assert.equal(reply.target, 'database_replica3', 'target db should match');
+          assert.equal(reply._replication_state, 'completed', 'replication should have completed');
+          replica3.list(function(error, list) {
+            assert.equal(error, null, 'should be able to invoke list');
+            assert.equal(list['total_rows'], 3, 'and have three documents');
+            db.replication.disable(reply._id, reply._rev, function(error, disabled) {
+              assert.true(disabled, 'should not be null');
+              assert.true(disabled.ok, 'should have stopped the replication');
+              assert.end();
+            });
+          });  
+        })
+      }, 
+      3000)
+    };
+    waitForReplication();
+  });
+});
+
+it('should destroy the extra databases', function(assert) {
+ async.forEach(['database_replica', 'database_replica2', 'database_replica3'],
+ nano.db.destroy, function(error) {
+   assert.equal(error, undefined, 'deleted databases');
+   assert.end();
+ });
+});
diff --git a/tests/unit/database/replicator.js b/tests/unit/database/replicator.js
@@ -0,0 +1,38 @@
+// Licensed under the Apache License, Version 2.0 (the 'License'); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+'use strict';
+
+var replicator = require('../../helpers/unit').unit([
+  'database',
+  'replicator'
+]);
+
+replicator('baa', 'baashep', {
+  body: '{"source":"baa","target":"baashep"}',
+  headers: {
+    accept: 'application/json',
+    'content-type': 'application/json'
+  },
+  method: 'POST',
+  uri: '/_replicator'
+});
+
+replicator('molly', 'anne', {some: 'params'}, {
+  body: '{"some":"params","source":"molly","target":"anne"}',
+  headers: {
+    accept: 'application/json',
+    'content-type': 'application/json'
+  },
+  method: 'POST',
+  uri: '/_replicator'
+});