finish postgres

README.md

@@ -154,7 +154,7 @@ api.post('/hook', function (req, res) {
 We setup our crontab to continuously look for jobs that have not yet been completed.
 
 ```bash
-* * * * * node $(npm bin -g)/pdf-bot -c ./pdf-bot.config.js shift >> /var/log/pdfbot.log 2>&1
+* * * * * node $(npm bin -g)/pdf-bot -c ./pdf-bot.config.js shift:all >> /var/log/pdfbot.log 2>&1
 * * * * * node $(npm bin -g)/pdf-bot -c ./pdf-bot.config.js ping:retry-failed >> /var/log/pdfbot.log 2>&1
 ```
 
@@ -166,10 +166,10 @@ Let us assume I want to generate a PDF for `https://esbenp.github.io`. I can add
 $ pdf-bot -c ./pdf-bot.config.js push https://esbenp.github.io --meta '{"id":1}'
 ```
 
-Next, if my crontab is not setup to run it automatically I can run it using the `shift` command
+Next, if my crontab is not setup to run it automatically I can run it using the `shift:all` command
 
 ```bash
-$ pdf-bot -c ./pdf-bot.config.js shift
+$ pdf-bot -c ./pdf-bot.config.js shift:all
 ```
 
 This will look for the oldest uncompleted job and run it.
@@ -283,6 +283,54 @@ curl -X POST -H 'Authorization: Bearer api-token' -H 'Content-Type: application/
   }'
 ```
 
+## Database
+
+### LowDB (file-database) (default)
+
+If you have low conurrency (run a job every now and then) you can use the default database driver that uses LowDB.
+
+```javascript
+var LowDB = require('pdf-bot/src/db/lowdb')
+
+module.exports = {
+  api: {
+    token: 'api-token'
+  },
+  db: LowDB({
+    lowDbOptions: {},
+    path: '' // defaults to $storagePath/db/db.json
+  }),
+  webhook: {
+    secret: '1234',
+    url: 'http://localhost:3000/webhooks/pdf'
+  }
+}
+```
+
+### PostgreSQL
+
+```javascript
+var pgsql = require('pdf-bot/src/db/pgsql')
+
+module.exports = {
+  api: {
+    token: 'api-token'
+  },
+  db: pgsql({
+    database: 'pdfbot',
+    username: 'pdfbot',
+    password: 'pdfbot',
+    port: 5432
+  }),
+  webhook: {
+    secret: '1234',
+    url: 'http://localhost:3000/webhooks/pdf'
+  }
+}
+```
+
+To install the necessary database tables, run `db:migrate`. You can also destroy the database by running `db:destroy`.
+
 ## Storage
 
 Currently `pdf-bot` comes bundled with build-in support for storing PDFs on Amazon S3.
@@ -335,6 +383,7 @@ module.exports = {
     // The token used to validate requests to your API. Not required, but 100% recommended.
     token: 'api-token'
   },
+  db: LowDB(), // see other drivers under Database
   // html-pdf-chrome
   generator: {
     // Triggers that specify when the PDF should be generated
@@ -351,20 +400,16 @@ module.exports = {
     // How many times should pdf-bot try to generate a PDF?
     // (default: 5)
     generationMaxTries: 5,
+    // How many generations to run at the same time when using shift:all
+    parallelism: 4,
     // How frequent should pdf-bot retry failed webhook pings?
     // (default: 1 min, 3 min, 10 min, 30 min, 60 min)
     webhookRetryStrategy: function(job, retries) {
       return decaySchedule[retries - 1] ? decaySchedule[retries - 1] : 0
     },
     // How many times should pdf-bot try to ping a webhook?
     // (default: 5)
-    webhookMaxTries: 5,
-    // In what path should the database be stored?
-    path: 'storage/db/db.json',
-    // pdf-bot uses lowdb. You can pass options to it here.
-    lowDbOptions: {
-
-    }
+    webhookMaxTries: 5
   },
   storage: {
     's3': createS3Config({
@@ -412,6 +457,8 @@ $ pdf-bot.js --config ./examples/pdf-bot.config.js --help
   Commands:
 
     api                   Start the API
+    db:migrate
+    db:destroy
     install
     generate [jobID]      Generate PDF for job
     jobs [options]        List all completed jobs
@@ -421,6 +468,7 @@ $ pdf-bot.js --config ./examples/pdf-bot.config.js --help
     purge [options]       Will remove all completed jobs
     push [options] [url]  Push new job to the queue
     shift                 Run the next job in the queue
+    shift:all             Run all unfinished jobs in the queue
 ```
 
 ## Debug mode

src/api.js

@@ -23,17 +23,18 @@ function createApi(createQueue, options = {}) {
       return
     }
 
-    var response = queue.addToQueue({
-      url: req.body.url,
-      meta: req.body.meta || {}
-    })
-
-    if (error.isError(response)) {
-      res.status(422).json(response)
-      return
-    }
-
-    res.status(201).json(response)
+    queue
+      .addToQueue({
+        url: req.body.url,
+        meta: req.body.meta || {}
+      }).then(function (response) {
+        if (error.isError(response)) {
+          res.status(422).json(response)
+          return
+        }
+
+        res.status(201).json(response)
+      })
   })
 
   return api

src/db/pgsql.js

@@ -90,11 +90,11 @@ function getById (db, id) {
 function getList (db, failed = false, completed = false, limit) {
   var query = 'SELECT * FROM jobs WHERE (completed_at is null AND jsonb_array_length(generations) = 0) '
 
-  if (!failed) {
+  if (failed) {
     query += ' OR (completed_at is null AND jsonb_array_length(generations) > 0)'
   }
 
-  if (!completed) {
+  if (completed) {
     query += ' OR (completed_at is not null)'
   }
 

src/queue.js

@@ -115,11 +115,14 @@ function processJob (db, generator, job, webhookOptions) {
                 return response
               }
 
-              // Important to return promise otherwise the npm cli process will exit early
-              return attemptPing(db, job, webhookOptions)
-                .then(function() {
-                  return response
-                })
+              // Re-fetch the job as storage has been added
+              return getById(db, job.id).then(function (job) {
+                // Important to return promise otherwise the npm cli process will exit early
+                return attemptPing(db, job, webhookOptions)
+                  .then(function() {
+                    return response
+                  })
+              })
             })
           }