Automate reading export file (#53)

https://eaflood.atlassian.net/jira/software/projects/CMEA/boards/907?selectedIssue=CMEA-282

In `cypress/integration/common/read_export.js` we create an example `When` step which can read a given export file from an S3 bucket, and example `Then` steps which check the value of the item at a given row/column, or check that a value exists in a specified column.

In `cypress/integration/legacy/cfd/cfd_steps.js` we update the `I see confirmation the transaction file is queued for export` step to capture the name of the queued file, and create `I log the transaction filename to prove it can be used in another step` as an example of how we access it from another step.

⚠️ Note that the environment variable `CYPRESS_S3_PATH` (the path we upload import files to) is now changed to `CYPRESS_S3_UPLOAD_PATH` so will need to be updated in existing environment files. We have created an additional `CYPRESS_S3_DOWNLOAD_PATH` which is the path we will download export files from.


* Rename existing S3 env var

We currently have an environment variable `CYPRESS_S3_PATH`, which we use to specify the folder files should be uploaded to. Since we want to be able to specify separate paths for uploading and downloading, we therefore rename this to `CYPRESS_S3_UPLOAD_PATH`

* Create initial `s3Download` task

We start by creatin a basic `s3Download` task which simply reads the specified file and logs its contents to the Cypress console.

* Complete feature

We develop the read export feature to take the file data returned by `s3Download` and split it into a 2-dimensional array. We can then retrieve the item at a given row and column.

* Edit comment

* Write `column contains` step

We write a step which allows us to check that the specified column has a particular value somewhere in it

* Develop legacy cfd steps

In the legacy cfd tests, we update the export file confirmation step to retrieve the filename and store it, and create an additional step which demonstrates how to retrieve the stored filename

* Update `read_export` to use `cy.wrap`

Having realised that `cy.wrap` is preferable to defining variables, we upddate the steps in `read_export`

* Add `CYPRESS_S3_DOWNLOAD_PATH` to `.example.env`

cypress/integration/common/import.js

@@ -6,7 +6,7 @@ Given('I import the file {string}', (filename) => {
       cy.task('s3Upload', {
         Body,
         Bucket: Cypress.env('S3_BUCKET'),
-        remotePath: Cypress.env('S3_PATH'),
+        remotePath: Cypress.env('S3_UPLOAD_PATH'),
         filename
       })
     })

cypress/integration/common/read_export.js

@@ -0,0 +1,39 @@
+import { Then, When } from 'cypress-cucumber-preprocessor/steps'
+
+When('I read the export file {string}', (filePath) => {
+  cy.task('s3Download', {
+    Bucket: Cypress.env('S3_BUCKET'),
+    remotePath: Cypress.env('S3_DOWNLOAD_PATH'),
+    filePath
+  }).then(data => {
+    cy.wrap(splitData(data)).as('exportData')
+  })
+})
+
+Then('row {int} column {int} equals {string}', (row, column, value) => {
+  cy.get('@exportData')
+    .then(exportData => expect(exportData[row][column]).to.equal(value))
+})
+
+Then('column {int} contains {string}', (column, value) => {
+  cy.get('@exportData')
+    .then(exportData => {
+      const columnData = exportData.map(row => row[column])
+      expect(columnData).to.include(value)
+    })
+})
+
+/**
+ * Splits the data into a two-dimensional array, removing the double quotes that normally enclose each item.
+ */
+function splitData (data, row, column) {
+  return data
+    // Split the data into an array of lines
+    .split('\n')
+    // Split each line into an array of items, and use regex replace to remove enclosing quotes from each item.
+    // https://thispointer.com/remove-first-and-last-double-quotes-from-a-string-in-javascript/
+    .map(line => line
+      .split(',')
+      .map(item => item.replace(/(^"|"$)/g, ''))
+    )
+}

cypress/integration/legacy/cfd.feature

@@ -60,6 +60,7 @@ Feature: CFD (Water Quality) Legacy
     And approve the transactions for billing
     And generate the transaction file
     Then I see confirmation the transaction file is queued for export
+    And I log the transaction filename to prove it can be used in another step
     And there are no transactions to be billed displayed anymore
     And I select 'Transaction File History' from the Transactions menu
     And the main heading is 'Transaction File History'

cypress/integration/legacy/cfd/cfd_steps.js

@@ -302,7 +302,26 @@ And('generate the transaction file', () => {
 })
 
 Then('I see confirmation the transaction file is queued for export', () => {
-  cy.get('div.alert-success.alert-dismissable').should('contain.text', 'Successfully queued')
+  cy.get('div.alert-success.alert-dismissable')
+    .should('contain.text', 'Successfully queued')
+    .then(alert => {
+      cy.wrap(getExportFilename(alert)).as('exportFilename')
+    })
+})
+
+function getExportFilename (element) {
+  return element
+    // Get the element text
+    .text()
+    // Use regex match to extract the text between "transaction file " and " for export"
+    // ?<= and ?= stop the matcher from returning those strings, so we just get the filename
+    // This results in an array with a single item, so we return that using [0]
+    .match(/(?<=transaction file )(.*)(?= for export)/g)[0]
+}
+
+And('I log the transaction filename to prove it can be used in another step', () => {
+  cy.get('@exportFilename')
+    .then(filename => cy.log(filename))
 })
 
 And('I set region to {word}', (option) => {

cypress/integration/read_export.feature

@@ -0,0 +1,7 @@
+Feature: Read Export Files
+
+  Scenario: Read export file
+    When I read the export file 'wrls/transaction/nalwi50001.dat'
+    Then row 0 column 0 equals 'H'
+    Then row 1 column 1 equals '0000001'
+    Then column 2 contains 'A51541393A'

cypress/plugins/index.js

@@ -15,7 +15,7 @@
 // Our custom config handler
 // import EnvironmentConfig from '../../config/index'
 
-const { S3Client, PutObjectCommand } = require('@aws-sdk/client-s3')
+const { S3Client, GetObjectCommand, PutObjectCommand } = require('@aws-sdk/client-s3')
 
 // Added to support use of Cucumber features
 const cucumber = require('cypress-cucumber-preprocessor').default
@@ -53,6 +53,21 @@ function loadDotenvPlugin (config) {
   return dotenvPlugin(config, { path: pathToEnvFile }, true)
 }
 
+/**
+ * We use GetObjectCommand to read data from an S3 bucket. This returns a readable stream of data, which we need to
+ * read in chunks then convert to a string for it to be usable.
+ * https://github.com/aws/aws-sdk-js-v3/issues/1877
+ * https://stackoverflow.com/questions/10623798/how-do-i-read-the-contents-of-a-node-js-stream-into-a-string-variable
+ */
+function streamToString (stream) {
+  const chunks = []
+  return new Promise((resolve, reject) => {
+    stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)))
+    stream.on('error', (err) => reject(err))
+    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')))
+  })
+}
+
 /**
  * @type {Cypress.PluginConfig}
  */
@@ -84,6 +99,32 @@ module.exports = (on, config) => {
               reject(error)
             })
       })
+    },
+
+    s3Download ({ Bucket, remotePath, filePath }) {
+      // We use a template literal to combine the paths rather than path.join() to ensure it joins them with a forward
+      // slash as required by S3 (which wouldn't happen if running under Windows).
+      const Key = `${remotePath}/${filePath}`
+      const client = new S3Client()
+      const command = new GetObjectCommand({ Bucket, Key })
+
+      return new Promise((resolve, reject) => {
+        client
+          .send(command)
+          .then(
+            // If client.send() was successful then resolve the promise so Cypress can continue.
+            response => {
+              // GetObjectCommand gives us a stream of data, which we convert into a string and then return.
+              streamToString(response.Body)
+                .then(data => {
+                  resolve(data)
+                })
+            },
+            // If client.send() failed then reject the promise so Cypress can throw an error
+            error => {
+              reject(error)
+            })
+      })
     }
   })
 

environments/.example.env

@@ -15,4 +15,5 @@ AWS_REGION=eu-west-1
 AWS_ACCESS_KEY_ID=access_key_id
 AWS_SECRET_ACCESS_KEY=secret_access_key
 CYPRESS_S3_BUCKET=file-uploads-dev-sroc-service-gov-uk
-CYPRESS_S3_PATH=import
+CYPRESS_S3_UPLOAD_PATH=import
+CYPRESS_S3_DOWNLOAD_PATH=export