Merge pull request #36 from fecgov/release/sprint-4

Release/sprint 4

.circleci/config.yml

@@ -38,15 +38,7 @@ jobs:
       - run:
           name: Run tests, save a coverage report, and save coverage percentage
           command: |
-            pytest --cov=.  --cov-report=xml --cov-report=html --cov-report=term || EXIT_CODE=$?
-            if [ $EXIT_CODE == "0" ]; then
-              echo "Tests ran successfully"
-            elif [ $EXIT_CODE == "5" ]; then
-              echo "No tests found. Continuing."
-            else
-              echo "Unit test errors. pytest exited with $EXIT_CODE."
-              exit 1
-            fi
+            PYTHONPATH=. pytest --cov=.  --cov-report=xml --cov-report=html --cov-report=term
 
       - store_artifacts:
           path: htmlcov
@@ -85,6 +77,9 @@ jobs:
           key: v1-sonarcloud-scanner-4.6.2.2472
           paths: /tmp/cache/scanner
 
+      - run:
+          name: flake8 lint tests
+          command: flake8 --config .flake8
   dependency-check:
     docker:
       - image: cimg/python:3.8
@@ -116,9 +111,34 @@ jobs:
             echo "----------------------------------------------------"
             eval $command
 
+  deploy:
+
+    docker:
+      - image: cimg/python:3.7
+
+    steps:
+      - checkout
+
+      - python/install-packages:
+          pkg-manager: pip
+          pip-dependency-file: requirements.txt
+
+      - run:
+          name: Installs for deploy
+          command: |
+            mkdir -p $HOME/bin
+            export PATH=$HOME/bin:$PATH
+            curl -L "https://cli.run.pivotal.io/stable?release=linux64-binary&version=7.1.0" | tar xzv -C $HOME/bin
+      - deploy:
+          name: Deploy API
+          command: |
+            export PATH=$HOME/bin:$PATH
+            invoke deploy --branch $CIRCLE_BRANCH --login
+
 # See: https://circleci.com/docs/2.0/configuration-reference/#workflows
 workflows:
   test:
     jobs:
       - test
-      - dependency-check
+      - dependency-check
+      - deploy

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 160
+ignore =
+    W503  # line break before binary operator

.gitignore

@@ -58,11 +58,12 @@ testem.log
 *.bower-registry
 *.bower-tmp
 
-
-
 # System Files
 .DS_Store
 Thumbs.db
 .vscode/settings.json
 front-end/.vscode/launch.json
 *pycache*
+
+# cloud foundry
+.cfmeta

Procfile

@@ -0,0 +1 @@
+web: bin/run.sh

bin/run.sh

@@ -0,0 +1 @@
+gunicorn --bind 0.0.0.0:8080 wsgi:APP -w 5 -t 200

bin/update-licence-report.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+CURRENT_DIR=${PWD##*/}
+OUTFILE="./docs/license_report.md"
+
+echo "Saving report as a markdown formatted file in ${OUTFILE}"
+
+if [ "$CURRENT_DIR" != "fecfile-image-generator" ]; then
+  echo "Run this script from the root directory of the fecfile-image-generator repository:"
+fi;
+
+mkdir -p docs
+
+echo "The following dependencies are used by this application:"  > "$OUTFILE"
+echo ""                                                          >> "$OUTFILE"
+echo ""                                                          >> "$OUTFILE"
+echo "| Package | Version | License |"                           >> "$OUTFILE"
+echo "| --- | --- | --- |"                                       >> "$OUTFILE"
+
+liccheck |
+  grep ": " |
+  sed "s/://" |
+  sed "s/\[//" |
+  sed "s/\]//" |
+  sed "s/'//g" |
+  awk '{
+        gsub(/\(/, "", $2)
+        gsub(/\)/, "", $2)
+        printf "| %s | %s | ", $1, $2
+        for (i=3; i <= NF; i++) printf "%s ", $i
+        printf "|\n"
+        }'  >> "$OUTFILE"
+
+  echo "" >> "$OUTFILE"
+
+
+
+
+

config.py

@@ -16,10 +16,10 @@
 ALLOWED_HOSTS = ["*"]
 
 
-FORM_TEMPLATES_LOCATION = 'templates/forms/{}.pdf'
-HTML_FORM_TEMPLATES_LOCATION = 'templates/forms/F99/{}.html'
-FORMS_LOCATION = 'templates/forms/{}'
-REQUEST_FILE_LOCATION = 'temp/json/{}.json'
+FORM_TEMPLATES_LOCATION = "templates/forms/{}.pdf"
+HTML_FORM_TEMPLATES_LOCATION = "templates/forms/F99/{}.html"
+FORMS_LOCATION = "templates/forms/{}"
+REQUEST_FILE_LOCATION = "temp/json/{}.json"
 # OUTPUT_FILE_LOCATION = 'output/pdf/{}.pdf'
 OUTPUT_DIR_LOCATION = "output/pdf/{}/"
 
@@ -33,16 +33,14 @@
 # AWS_SECRET_ACCESS_KEY = os.environ.get('SECRET_KEY', None)
 # AWS_HOST_NAME = 'us-east-1'
 # AWS_REGION = 'us-east-1'
-AWS_SES_AUTO_THROTTLE = (
-    0.5
-)  # (default; safety factor applied to rate limit, turn off automatic throttling, set this to None)
+AWS_SES_AUTO_THROTTLE = 0.5  # (default; safety factor applied to rate limit, turn off automatic throttling, set this to None)
 
 # AWS FECFile components bucket name
 
-AWS_FECFILE_OUTPUT_DIRECTORY = os.environ.get('OUTPUT_DIRECTORY', 'output')
+AWS_FECFILE_OUTPUT_DIRECTORY = os.environ.get("OUTPUT_DIRECTORY", "output")
 AWS_FECFILE_COMPONENTS_BUCKET_NAME = "fecfile-dev-components"
-AWS_FECFILE_TEMP_BUCKET_NAME = os.environ.get('TEMP_BUCKET', 'dev-efile-repo')
-AWS_FECFILE_PDF_BUCKET_NAME = os.environ.get('PERM_BUCKET', 'fecfile-pdf')
+AWS_FECFILE_TEMP_BUCKET_NAME = os.environ.get("TEMP_BUCKET", "dev-efile-repo")
+AWS_FECFILE_PDF_BUCKET_NAME = os.environ.get("PERM_BUCKET", "fecfile-pdf")
 
 
 # if False it will create unique file names for every uploaded file
@@ -56,28 +54,40 @@
     "%s.s3.amazonaws.com" % AWS_FECFILE_TEMP_BUCKET_NAME
 )
 
-S3_FILE_URL = "https://%s/%s/" % (AWS_S3_FECFILE_COMPONENTS_DOMAIN, AWS_FECFILE_OUTPUT_DIRECTORY)
+S3_FILE_URL = "https://%s/%s/" % (
+    AWS_S3_FECFILE_COMPONENTS_DOMAIN,
+    AWS_FECFILE_OUTPUT_DIRECTORY,
+)
 
 # the sub-directories of temp and output files
 # TEMP_FILES_LOCATION = 'temp'
 # OUTPUT_FILE_FOLDER = 'output'
 
 # TEMP_FILES_URL = "https://%s/%s/{}" % (AWS_S3_FECFILE_COMPONENTS_DOMAIN, TEMP_FILES_LOCATION)
-PRINT_OUTPUT_FILE_URL = "https://%s/%s" % (AWS_S3_FECFILE_COMPONENTS_DOMAIN, OUTPUT_DIR_LOCATION)
+PRINT_OUTPUT_FILE_URL = "https://%s/%s" % (
+    AWS_S3_FECFILE_COMPONENTS_DOMAIN,
+    OUTPUT_DIR_LOCATION,
+)
 
-FECFILE_UTIL_PRINT_API_URL = os.environ.get('FECFILE_UTIL_URL', 'https://dev-efile-api.efdev.fec.gov/printpdf')
+FECFILE_UTIL_PRINT_API_URL = os.environ.get(
+    "FECFILE_UTIL_URL", "https://dev-efile-api.efdev.fec.gov/printpdf"
+)
 FECFILE_UTIL_API_VERSION = "/v1/fecfileutil"
 
-NXG_FEC_PARSER_API_URL = os.environ.get('PARSER_URL', 'https://dev-efile-api.efdev.fec.gov/receiver')
+NXG_FEC_PARSER_API_URL = os.environ.get(
+    "PARSER_URL", "https://dev-efile-api.efdev.fec.gov/receiver"
+)
 # NXG_FEC_PARSER_API_URL = os.environ.get('PARSER_URL', 'http://localhost:8090')
 NXG_FEC_PARSER_API_VERSION = "/v1"
 
 
 # SQS Details
-DATA_RECEIVER_SQS_QUEUE = os.environ.get('DATA_RECEIVER_SQS_QUEUE', 'datareceiver-dev')
-VALIDATION_SQS_QUEUE = os.environ.get('VALIDATION_SQS_QUEUE', 'validator-dev')
-IMAGE_NUMBER_SQS_QUEUE = os.environ.get('IMAGE_NUMBER_SQS_QUEUE', 'imaging')
+DATA_RECEIVER_SQS_QUEUE = os.environ.get("DATA_RECEIVER_SQS_QUEUE", "datareceiver-dev")
+VALIDATION_SQS_QUEUE = os.environ.get("VALIDATION_SQS_QUEUE", "validator-dev")
+IMAGE_NUMBER_SQS_QUEUE = os.environ.get("IMAGE_NUMBER_SQS_QUEUE", "imaging")
 # IMAGE_NUMBER_SQS_QUEUE = os.environ.get('IMAGE_NUMBER_SQS_QUEUE', 'validator-dev')
-IMAGE_GENERATOR_SQS_QUEUE = os.environ.get('IMAGE_GENERATOR_SQS_QUEUE', 'imaging-generator')
+IMAGE_GENERATOR_SQS_QUEUE = os.environ.get(
+    "IMAGE_GENERATOR_SQS_QUEUE", "imaging-generator"
+)
 # Timeout for the message in queue, after the set time interval other process will see it
 MESSAGE_VISIBILITY_TIMEOUT = 30  # setting it to 0 for testing

liccheck.ini

@@ -0,0 +1,7 @@
+# Authorized and unauthorized licenses in LOWER CASE
+[Licenses]
+authorized_licenses:
+
+unauthorized_licenses:
+
+

manifest-dev.yml

@@ -0,0 +1,9 @@
+---
+applications:
+- name: fecfile-image-generator
+  routes:
+  - route: fecfile-image-generator-dev.app.cloud.gov
+  stack: cflinuxfs3
+  buildpacks:
+    - python_buildpack
+  memory: 512M

manifest-prod.yml

@@ -0,0 +1,9 @@
+---
+applications:
+- name: fecfile-image-generator
+  routes:
+  - route: fecfile-image-generator-prod.app.cloud.gov
+  stack: cflinuxfs3
+  buildpacks:
+    - python_buildpack
+  memory: 512M

manifest-stage.yml

@@ -0,0 +1,9 @@
+---
+applications:
+- name: fecfile-image-generator
+  routes:
+  - route: fecfile-image-generator-stage.app.cloud.gov
+  stack: cflinuxfs3
+  buildpacks:
+    - python_buildpack
+  memory: 512M

requirements-test.txt

@@ -1,3 +1,5 @@
 pytest==6.2.5
 pytest-cov==3.0.0
-safety==1.10.3
+safety==1.10.3
+flake8==4.0.1
+liccheck

requirements.txt

@@ -1,30 +1,27 @@
-BeautifulSoup4==4.7.1
+beautifulsoup4==4.7.1
+boto3==1.21.2
 certifi==2018.10.15
+cfenv==0.5.2
 chardet==3.0.4
-Click==7.0
-cx-Oracle==7.0.0
-Flask==1.0.2
+Flask==2.0.2
+Flask-API==3.0.post1
 Flask-Cors==3.0.9
 Flask-HTTPAuth==3.2.4
+Flask==2.0.2
+GitPython==3.1.0
+gunicorn==19.10.0
 idna==2.7
-itsdangerous==0.24
-Jinja2==2.11.3
-MarkupSafe==1.1.1
+invoke==0.15.0
+ipdb==0.12.3
+itsdangerous==2.1.0
+Jinja2==3.0
+MarkupSafe==2.0.1
 passlib==1.7.1
+pdfkit==0.6.1
+PyPDF2==1.26.0
+pypdftk==0.4
 python-json-logger==0.1.9
-requests==2.22.0
+requests==2.27.1
 six==1.11.0
-urllib3==1.25.3
-Werkzeug==2.0.2
+urllib3==1.26.8
 WeasyPrint==47
-ipdb==0.12.3
-pypdftk==0.4
-boto3==1.14.60
-botocore==1.17.60
-Flask-API==1.1
-PyPDF2==1.26.0
-pdfkit==0.6.1
-beautifulsoup4==4.7.1
-
-gunicorn==19.10.0
-flake8==3.6.0

routes/src/common.py

@@ -49,8 +49,8 @@ def check_response_and_return_or_log(response, url):
                     return True, response_text.get("data")
             except Exception as e:
                 LOGGER.error(
-                    u'Exception "%s" raised when trying to read '
-                    u"response.text as json" % str(e)
+                    'Exception "%s" raised when trying to read '
+                    "response.text as json" % str(e)
                 )
                 # if response.text does not indicate failure assume
                 # success = True
@@ -61,13 +61,13 @@ def check_response_and_return_or_log(response, url):
 
     elif 400 <= response.status_code < 500:
         LOGGER.error(
-            u"%s Bouncer Service Error: %s for url: %s"
+            "%s Bouncer Service Error: %s for url: %s"
             % (response.status_code, response.text, url)
         )
 
     elif 500 <= response.status_code < 600:
         LOGGER.error(
-            u"%s Server Error: %s for url: %s"
+            "%s Server Error: %s for url: %s"
             % (response.status_code, response.text, url)
         )
 

routes/src/controllers.py

@@ -19,6 +19,12 @@
 CORS(app)
 
 
+@app.route("/", methods=["GET"])
+@app.route("/app-name", methods=["GET"])
+def index():
+    return "fecfile-image-generator"
+
+
 @app.route("/print", methods=["POST"])
 def print_pdf():
     """
@@ -359,7 +365,7 @@ def image_number_data(next_imaging=None):
             "submissionId": submission_id,
             "totalPages": total_pages,
         }
-        ## data_obj = json.dumps(data_obj)
+        # data_obj = json.dumps(data_obj)
         begin_image_number_object = requests.post(
             cfg.NXG_FEC_PARSER_API_URL
             + cfg.NXG_FEC_PARSER_API_VERSION
@@ -487,7 +493,7 @@ def image_generator_data(next_image_generator=None):
     json_file_name = next_image_generator["fileName"]
     begin_image_number = next_image_generator["beginImageNumber"]
     filing_timestamp = next_image_generator["receivedTime"]
-    rep_id = json_file_name[0 : json_file_name.index(".json")]
+    rep_id = json_file_name[0:json_file_name.index(".json")]
     # rep_id = '8'
     # print(rep_id)