From b02222f10d245c55d6b268f5eafe3d7951a07bd5 Mon Sep 17 00:00:00 2001 From: dzaslavskiy Date: Thu, 11 May 2023 11:05:21 -0400 Subject: [PATCH] initial commit --- .bandit | 2 ++ .cfignore | 10 ++++++ .codeclimate.yml | 5 +++ .gitignore | 2 ++ .pre-commit-config.yaml | 12 +++++++ CONTRIBUTING.md | 37 ++++++++++++++++++++++ LICENSE.md | 33 +++++++++++++++++++ SECURITY.md | 32 +++++++++++++++++++ manifest.yaml | 12 +++++++ qualtrix/__init__.py | 0 qualtrix/api.py | 34 ++++++++++++++++++++ qualtrix/client.py | 70 +++++++++++++++++++++++++++++++++++++++++ qualtrix/main.py | 18 +++++++++++ qualtrix/settings.py | 20 ++++++++++++ requirements-dev.txt | 6 ++++ requirements.txt | 6 ++++ vars.yaml | 3 ++ 17 files changed, 302 insertions(+) create mode 100644 .bandit create mode 100644 .cfignore create mode 100644 .codeclimate.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE.md create mode 100644 SECURITY.md create mode 100644 manifest.yaml create mode 100644 qualtrix/__init__.py create mode 100644 qualtrix/api.py create mode 100644 qualtrix/client.py create mode 100644 qualtrix/main.py create mode 100644 qualtrix/settings.py create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100644 vars.yaml diff --git a/.bandit b/.bandit new file mode 100644 index 0000000..008a49a --- /dev/null +++ b/.bandit @@ -0,0 +1,2 @@ +[bandit] +exclude: qualtrix/tests, .venv/ diff --git a/.cfignore b/.cfignore new file mode 100644 index 0000000..c8f3eef --- /dev/null +++ b/.cfignore @@ -0,0 +1,10 @@ +*.md +.venv/ +qualtrix/__pycache__/ +.bandit +.codeclimate.yml +.github +.pre-commit-config.yaml +requirements-dev.txt +tests/ +vars.yaml diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 0000000..1240cdf --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,5 @@ +--- +version: "2" +plugins: + bandit: + enabled: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..033df5f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.venv +__pycache__ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..b3998f3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +--- +repos: + - repo: https://github.com/psf/black + rev: 22.3.0 # Update with 'pre-commit autoupdate' + hooks: + - id: black + + - repo: https://github.com/PyCQA/bandit + rev: 1.7.4 + hooks: + - id: bandit + exclude: tests diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..419ae8f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,37 @@ +# Welcome! + +We're so glad you're thinking about contributing to a +[open source project of the U.S. government](https://code.gov/)! If you're +unsure about anything, just ask -- or submit the issue or pull request anyway. +The worst that can happen is you'll be politely asked to change something. We +love all friendly contributions. + +We encourage you to read this project's CONTRIBUTING policy (you are here), its +[LICENSE](LICENSE.md), and its [README](README.md). + +## Policies + +We want to ensure a welcoming environment for all of our projects. Our staff +follow the [TTS Code of Conduct](https://18f.gsa.gov/code-of-conduct/) and +all contributors should do the same. + +We adhere to the +[18F Open Source Policy](https://github.com/18f/open-source-policy). If you +have any questions, just [shoot us an email](mailto:18f@gsa.gov). + +As part of a U.S. government agency, the General Services Administration +(GSA)’s Technology Transformation Services (TTS) takes seriously our +responsibility to protect the public’s information, including financial and +personal information, from unwarranted disclosure. For more information about +security and vulnerability disclosure for our projects, please read our +[18F Vulnerability Disclosure Policy](https://18f.gsa.gov/vulnerability-disclosure-policy/). + +## Public domain + +This project is in the public domain within the United States, and copyright +and related rights in the work worldwide are waived through the +[CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). + +All contributions to this project will be released under the CC0 dedication. By +submitting a pull request or issue, you are agreeing to comply with this waiver +of copyright interest. diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..5f3ccbe --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,33 @@ +# License + +As a work of the [United States government](https://www.usa.gov/), this project +is in the public domain within the United States of America. + +Additionally, we waive copyright and related rights in the work worldwide +through the CC0 1.0 Universal public domain dedication. + +## CC0 1.0 Universal Summary + +This is a human-readable summary of the +[Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode). + +### No Copyright + +The person who associated a work with this deed has dedicated the work to the +public domain by waiving all of their rights to the work worldwide under +copyright law, including all related and neighboring rights, to the extent +allowed by law. + +You can copy, modify, distribute, and perform the work, even for commercial +purposes, all without asking permission. + +### Other Information + +In no way are the patent or trademark rights of any person affected by CC0, nor +are the rights that other persons may have in the work or in how the work is +used, such as publicity or privacy rights. + +Unless expressly stated otherwise, the person who associated a work with this +deed makes no warranties about the work, and disclaims liability for all uses +of the work, to the fullest extent permitted by applicable law. When using or +citing the work, you should not imply endorsement by the author or the affirmer. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..aec2ce2 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,32 @@ +# Security Policy + +The GIVE team takes the security of our software seriously. If you believe +you have found a security vulnerability in any GIVE repository, please report +it to us as described below. + +## Supported Versions + +GIVE will only ever be providing security updates for the most recent +version of its software. This should always be available as the most recently +tagged version within this repository. We will not be providing security +updates to versions that are not currently released into production. + +## Reporting a Vulnerability + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them by emailing email give@gsa.gov. You should receive +a response within 72 hours. If for some reason you do not, please follow up via +email to ensure we've received your original message. + +Please include the requested information listed below, or as much as you can +provide, to help us better understand the nature and scope of the possible issue: + +* Issue type (e.g. buffer overflow, SQL injection, cross-site scripting, etc) +* Full paths of source file(s) related to the manifestation of the issue +* Location of the effected source code (direct URL or tag/branch/commit) +* Step-by-step instructions on how to reproduce the issue +* Proof-of-concept or exploit code (if possible) +* Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. diff --git a/manifest.yaml b/manifest.yaml new file mode 100644 index 0000000..afe5522 --- /dev/null +++ b/manifest.yaml @@ -0,0 +1,12 @@ +--- +applications: + - name: qualtrix + routes: + - route: idva-qualtrix-((ENVIRONMENT)).apps.internal + memory: ((MEMORY)) + instances: ((INSTANCES)) + buildpacks: + - python_buildpack + command: uvicorn qualtrix.main:app --host 0.0.0.0 --port $PORT +# services: +# - qualtrix diff --git a/qualtrix/__init__.py b/qualtrix/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/qualtrix/api.py b/qualtrix/api.py new file mode 100644 index 0000000..6f0116d --- /dev/null +++ b/qualtrix/api.py @@ -0,0 +1,34 @@ +""" +qualtrix rest api +""" + +import base64 as base64decoder +import io +import logging + +import fastapi +from fastapi import Body +from starlette.requests import Request + +from . import client, settings + +log = logging.getLogger(__name__) + +router = fastapi.APIRouter() + + +@router.post("/bulk-responses") +async def test(surveyId: str): + + return client.result_export() + +@router.post("/response") +async def test(responseId: str): + + client.get_response(responseId) + +@router.post("/survey-schema") +async def test(surveyId: str): + + return client.get_survey_schema(surveyId) + diff --git a/qualtrix/client.py b/qualtrix/client.py new file mode 100644 index 0000000..051a169 --- /dev/null +++ b/qualtrix/client.py @@ -0,0 +1,70 @@ +import io +from itertools import permutations +import mimetypes +import logging +import requests +import time + +log = logging.getLogger(__name__) + +# Permisions # read:survey_responses + +auth_header = {"X-API-TOKEN": settings.API_TOKEN} + +def get_response(response_id: str): + r = requests.get(settings.BASE_URL + f"/surveys/{settings.SURVEY_URL}/responses/{response_id}" ,headers=auth_header ) + + if r.status_code == 200: + return r.json() + +def get_survey_schema(): + r = requests.get(settings.BASE_URL + f"/surveys/{settings.SURVEY_URL}/response-schema" ,headers=auth_header ) + + if r.status_code == 200: + return r.json() + +def result_export(): + + r_body = {"format": "json", "compress": False, "sortByLastModifiedDate" : True} #, "startDate": "", "endDate": "" + + r = requests.post(settings.BASE_URL + f"/surveys/{settings.SURVEY_URL}/export-responses" ,headers=auth_header, json=r_body ) + + if r.status_code != 200: + print("error") + return + + progress_id = r.json()["result"]["progressId"] + + while True: + r = requests.get(settings.BASE_URL + f"/surveys/{settings.SURVEY_URL}/export-responses/{progress_id}" ,headers=auth_header ) + status = r.json()["result"]["status"] + + if status == "complete": + file_id = r.json()["result"]["fileId"] + break + if status == "failed": + break + if status == "inProgress": + time.sleep(1) + + r = requests.get(settings.BASE_URL + f"/surveys/{settings.SURVEY_URL}/export-responses/{file_id}/file" ,headers=auth_header ) + + results = r.json()['responses'] + answers = [] + for result in results: + try: + answer = { + "ethnicity": result["labels"]["QID12"], + "race": result["labels"]["QID36"], + "gender": result["labels"]["QID14"], + "age": result["values"]["QID15_TEXT"], + "browser": result["values"]["QID17_BROWSER"], + "version": result["values"]["QID17_VERSION"], + "os": result["values"]["QID17_OS"], + "resolution": result["values"]["QID17_RESOLUTION"] + } + answers.append(answer) + except KeyError: + pass + + return answers diff --git a/qualtrix/main.py b/qualtrix/main.py new file mode 100644 index 0000000..b412072 --- /dev/null +++ b/qualtrix/main.py @@ -0,0 +1,18 @@ +""" +Qualtrix Microservice FastAPI Web App. +""" +import logging + +import fastapi +import starlette_prometheus + +from . import api, settings + +logging.basicConfig(level=settings.LOG_LEVEL) + +app = fastapi.FastAPI() + +app.add_middleware(starlette_prometheus.PrometheusMiddleware) +app.add_route("/metrics/", starlette_prometheus.metrics) + +app.include_router(api.router) diff --git a/qualtrix/settings.py b/qualtrix/settings.py new file mode 100644 index 0000000..144b737 --- /dev/null +++ b/qualtrix/settings.py @@ -0,0 +1,20 @@ +""" +Configuration for the qualtrix microservice settings. +Context is switched based on if the app is in debug mode. +""" +import json +import logging +import os + +log = logging.getLogger(__name__) + + +# SECURITY WARNING: don't run with debug turned on in production! +# DEBUG set is set to True if env var is "True" +DEBUG = os.getenv("DEBUG", "False") == "True" + +LOG_LEVEL = os.getenv("LOG_LEVEL", logging.getLevelName(logging.INFO)) + +API_TOKEN = os.getenv("API_TOKEN") +BASE_URL = os.getenv("BASE_URL") +SURVEY_ID = os.getenv("SURVEY_ID") \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..7e408ad --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,6 @@ +-r requirements.txt +pre-commit +black +pylint +bandit +pytest diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2ac0bbc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.78.0 +uvicorn==0.17.6 +starlette-prometheus==0.9.0 +google-api-python-client==2.55.0 +google-auth-httplib2==0.1.0 +google-auth-oauthlib==0.5.2 diff --git a/vars.yaml b/vars.yaml new file mode 100644 index 0000000..d10ff05 --- /dev/null +++ b/vars.yaml @@ -0,0 +1,3 @@ +--- +INSTANCES: 2 +MEMORY: 128M