diff --git a/README.md b/README.md index bf0d811..6dc6ccf 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ api.post('/hook', function (req, res) { We setup our crontab to continuously look for jobs that have not yet been completed. ```bash -* * * * * node $(npm bin -g)/pdf-bot -c ./pdf-bot.config.js shift >> /var/log/pdfbot.log 2>&1 +* * * * * node $(npm bin -g)/pdf-bot -c ./pdf-bot.config.js shift:all >> /var/log/pdfbot.log 2>&1 * * * * * node $(npm bin -g)/pdf-bot -c ./pdf-bot.config.js ping:retry-failed >> /var/log/pdfbot.log 2>&1 ``` @@ -166,10 +166,10 @@ Let us assume I want to generate a PDF for `https://esbenp.github.io`. I can add $ pdf-bot -c ./pdf-bot.config.js push https://esbenp.github.io --meta '{"id":1}' ``` -Next, if my crontab is not setup to run it automatically I can run it using the `shift` command +Next, if my crontab is not setup to run it automatically I can run it using the `shift:all` command ```bash -$ pdf-bot -c ./pdf-bot.config.js shift +$ pdf-bot -c ./pdf-bot.config.js shift:all ``` This will look for the oldest uncompleted job and run it. @@ -283,6 +283,54 @@ curl -X POST -H 'Authorization: Bearer api-token' -H 'Content-Type: application/ }' ``` +## Database + +### LowDB (file-database) (default) + +If you have low conurrency (run a job every now and then) you can use the default database driver that uses LowDB. + +```javascript +var LowDB = require('pdf-bot/src/db/lowdb') + +module.exports = { + api: { + token: 'api-token' + }, + db: LowDB({ + lowDbOptions: {}, + path: '' // defaults to $storagePath/db/db.json + }), + webhook: { + secret: '1234', + url: 'http://localhost:3000/webhooks/pdf' + } +} +``` + +### PostgreSQL + +```javascript +var pgsql = require('pdf-bot/src/db/pgsql') + +module.exports = { + api: { + token: 'api-token' + }, + db: pgsql({ + database: 'pdfbot', + username: 'pdfbot', + password: 'pdfbot', + port: 5432 + }), + webhook: { + secret: '1234', + url: 'http://localhost:3000/webhooks/pdf' + } +} +``` + +To install the necessary database tables, run `db:migrate`. You can also destroy the database by running `db:destroy`. + ## Storage Currently `pdf-bot` comes bundled with build-in support for storing PDFs on Amazon S3. @@ -335,6 +383,7 @@ module.exports = { // The token used to validate requests to your API. Not required, but 100% recommended. token: 'api-token' }, + db: LowDB(), // see other drivers under Database // html-pdf-chrome generator: { // Triggers that specify when the PDF should be generated @@ -351,6 +400,8 @@ module.exports = { // How many times should pdf-bot try to generate a PDF? // (default: 5) generationMaxTries: 5, + // How many generations to run at the same time when using shift:all + parallelism: 4, // How frequent should pdf-bot retry failed webhook pings? // (default: 1 min, 3 min, 10 min, 30 min, 60 min) webhookRetryStrategy: function(job, retries) { @@ -358,13 +409,7 @@ module.exports = { }, // How many times should pdf-bot try to ping a webhook? // (default: 5) - webhookMaxTries: 5, - // In what path should the database be stored? - path: 'storage/db/db.json', - // pdf-bot uses lowdb. You can pass options to it here. - lowDbOptions: { - - } + webhookMaxTries: 5 }, storage: { 's3': createS3Config({ @@ -412,6 +457,8 @@ $ pdf-bot.js --config ./examples/pdf-bot.config.js --help Commands: api Start the API + db:migrate + db:destroy install generate [jobID] Generate PDF for job jobs [options] List all completed jobs @@ -421,6 +468,7 @@ $ pdf-bot.js --config ./examples/pdf-bot.config.js --help purge [options] Will remove all completed jobs push [options] [url] Push new job to the queue shift Run the next job in the queue + shift:all Run all unfinished jobs in the queue ``` ## Debug mode diff --git a/src/api.js b/src/api.js index d514f05..400ab24 100644 --- a/src/api.js +++ b/src/api.js @@ -23,17 +23,18 @@ function createApi(createQueue, options = {}) { return } - var response = queue.addToQueue({ - url: req.body.url, - meta: req.body.meta || {} - }) - - if (error.isError(response)) { - res.status(422).json(response) - return - } - - res.status(201).json(response) + queue + .addToQueue({ + url: req.body.url, + meta: req.body.meta || {} + }).then(function (response) { + if (error.isError(response)) { + res.status(422).json(response) + return + } + + res.status(201).json(response) + }) }) return api diff --git a/src/db/pgsql.js b/src/db/pgsql.js index a055c55..164cf19 100644 --- a/src/db/pgsql.js +++ b/src/db/pgsql.js @@ -90,11 +90,11 @@ function getById (db, id) { function getList (db, failed = false, completed = false, limit) { var query = 'SELECT * FROM jobs WHERE (completed_at is null AND jsonb_array_length(generations) = 0) ' - if (!failed) { + if (failed) { query += ' OR (completed_at is null AND jsonb_array_length(generations) > 0)' } - if (!completed) { + if (completed) { query += ' OR (completed_at is not null)' } diff --git a/src/queue.js b/src/queue.js index d94d2ef..4cedb59 100644 --- a/src/queue.js +++ b/src/queue.js @@ -115,11 +115,14 @@ function processJob (db, generator, job, webhookOptions) { return response } - // Important to return promise otherwise the npm cli process will exit early - return attemptPing(db, job, webhookOptions) - .then(function() { - return response - }) + // Re-fetch the job as storage has been added + return getById(db, job.id).then(function (job) { + // Important to return promise otherwise the npm cli process will exit early + return attemptPing(db, job, webhookOptions) + .then(function() { + return response + }) + }) }) }