From abcda87e2e14c9b6ac1936cd4b0b9bcc29c9afef Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 11 Mar 2019 08:52:57 -0400 Subject: [PATCH 001/253] First commit --- .flake8 | 18 +++++++ .gitignore | 2 + .pre-commit-config.yaml | 67 +++++++++++++++++++++++ .prettierignore | 5 ++ .travis.yml | 11 ++++ .yamllint | 7 +++ CONTRIBUTING.md | 109 +++++++++++++++++++++++++++++++++++++ LICENSE.md | 116 ++++++++++++++++++++++++++++++++++++++++ README.md | 35 ++++++++++++ requirements-dev.txt | 2 + requirements-test.txt | 3 ++ 11 files changed, 375 insertions(+) create mode 100644 .flake8 create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .prettierignore create mode 100644 .travis.yml create mode 100644 .yamllint create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 requirements-dev.txt create mode 100644 requirements-test.txt diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..33cdc3e --- /dev/null +++ b/.flake8 @@ -0,0 +1,18 @@ +[flake8] +max-line-length = 80 +# Select (turn on) +# * Complexity violations reported by mccabe (C) - +# http://flake8.pycqa.org/en/latest/user/error-codes.html#error-violation-codes +# * Default errors and warnings reported by pycodestyle (E and W) - +# https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes +# * Default errors reported by pyflakes (F) - +# http://flake8.pycqa.org/en/latest/glossary.html#term-pyflakes +# * Default warnings reported by flake8-bugbear (B) - +# https://github.com/PyCQA/flake8-bugbear#list-of-warnings +# * The B950 flake8-bugbear opinionated warning - +# https://github.com/PyCQA/flake8-bugbear#opinionated-warnings +select = C,E,F,W,B,B950 +# Ignore flake8's default warning about maximum line length, which has +# a hard stop at the configured value. Instead we use +# flake8-bugbear's B950, which allows up to 10% overage. +ignore = E501 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..073a081 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +.python-version diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d339352 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,67 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.1.0 + hooks: + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + - id: check-xml + - id: debug-statements + - id: detect-aws-credentials + args: + - --allow-missing-credentials + - id: detect-private-key + - id: end-of-file-fixer + exclude: files/(issue|motd) + - id: mixed-line-ending + args: + - --fix=lf + - id: pretty-format-json + args: + - --autofix + - id: requirements-txt-fixer + - id: trailing-whitespace + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: v0.14.0 + hooks: + - id: markdownlint + # The LICENSE.md must match the license text exactly for + # GitHub's autorecognition fu to work, so we should leave it + # alone. + exclude: LICENSE.md + - repo: https://github.com/adrienverge/yamllint + rev: v1.15.0 + hooks: + - id: yamllint + - repo: https://github.com/detailyang/pre-commit-shell + rev: 1.0.4 + hooks: + - id: shell-lint + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.7.7 + hooks: + - id: flake8 + - repo: https://github.com/asottile/pyupgrade + rev: v1.12.0 + hooks: + - id: pyupgrade + - repo: https://github.com/ambv/black + rev: 18.9b0 + hooks: + - id: black + - repo: https://github.com/ansible/ansible-lint.git + rev: v4.1.0a0 + hooks: + - id: ansible-lint + # files: molecule/default/playbook.yml + - repo: https://github.com/antonbabenko/pre-commit-terraform.git + rev: v1.11.0 + hooks: + - id: terraform_fmt + - id: terraform_validate_no_variables + - id: terraform_docs + - repo: https://github.com/prettier/prettier + rev: 1.16.2 + hooks: + - id: prettier diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..f1e840c --- /dev/null +++ b/.prettierignore @@ -0,0 +1,5 @@ +# Already being linted by mdl +*.md +# Already being linted by yamllint +*.yaml +*.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..028a17e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +--- +dist: xenial +language: python +python: 3.7 +services: docker + +install: + - pip install --upgrade -r requirements-test.txt +script: + - pre-commit run --all-files + - molecule test diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..7ed00eb --- /dev/null +++ b/.yamllint @@ -0,0 +1,7 @@ +--- +extends: default + +rules: + # yamllint doesn't like when we use yes and no for true and false, + # but that's pretty standard in Ansible. + truthy: disable diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a7a2f33 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,109 @@ +# Welcome # + +We're so glad you're thinking about contributing to this open source +project! If you're unsure or afraid of anything, just ask or submit +the issue or pull request anyway. The worst that can happen is that +you'll be politely asked to change something. We appreciate any sort +of contribution, and don't want a wall of rules to get in the way of +that. + +Before contributing, we encourage you to read our CONTRIBUTING policy +(you are here), our [LICENSE](LICENSE.md), and our +[README](README.md), all of which should be in this repository. + +## Issues ## + +If you want to report a bug or request a new feature, the most direct +method is to [create an +issue](https://github.com/cisagov/generic-skeleton/issues) in this +repository. We recommend that you first search through existing +issues (both open and closed) to check if your particular issue has +already been reported. If it has then you might want to add a comment +to the existing issue. If it hasn't then feel free to create a new +one. + +## Pull requests ## + +If you choose to [submit a pull +request](https://github.com/cisagov/generic-skeleton/pulls), you will +notice that our continuous integration (CI) system runs a fairly +extensive set of linters and syntax checkers. Your pull request may +fail these checks, and that's OK. If you want you can stop there and +wait for us to make the necessary corrections to ensure your code +passes the CI checks. + +If you want to make the changes yourself, or if you want to become a +regular contributor, then you will want to set up +[pre-commit](https://pre-commit.com/) on your local machine. Once you +do that, the CI checks will run locally before you even write your +commit message. This speeds up your development cycle considerably. + +### Setting up pre-commit ### + +There are a few ways to do this, but we prefer to use +[`pyenv`](https://github.com/pyenv/pyenv) and +[`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv) to +create and manage a Python virtual environment specific to this +project. + +#### Installing and using `pyenv` and `pyenv-virtualenv` #### + +On the Mac, installation is as simple as `brew install pyenv +pyenv-virtualenv` and adding this to your profile: + +```bash +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" +``` + +For Linux you will need to find a package for your particular distro, +or follow the manual instructions +[here](https://github.com/pyenv/pyenv#installation) and +[here](https://github.com/pyenv/pyenv-virtualenv#installation). Then +you will need to add the same bit above to your profile. + +For a list of Python versions that are already installed and ready to +use with `pyenv`, use the command `pyenv versions`. To see a list of +the Python versions available to be installed and used with `pyenv` +use the command `pyenv install --list`. You can read more +[here](https://github.com/pyenv/pyenv/blob/master/COMMANDS.md) about +the many things that `pyenv` can do. See +[here](https://github.com/pyenv/pyenv-virtualenv#usage) for the +additional capabilities that pyenv-virtualenv adds to the `pyenv` +command. + +#### Creating the Python virtual environment #### + +Once `pyenv` and `pyenv-virtualenv` are installed on your system, you +can create and configure the Python virtual environment with these +commands: + +```bash +cd generic-skeleton +pyenv virtualenv generic-skeleton +pyenv local generic-skeleton +pip install -r requirements-dev.txt +``` + +#### Installing the pre-commit hook #### + +Now setting up pre-commit is as simple as: + +```bash +pre-commit install +``` + +At this point the pre-commit checks will run against any files that +you attempt to commit. If you want to run the checks against the +entire repo, just execute `pre-commit run --all-files`. + +## Public domain ## + +This project is in the public domain within the United States, and +copyright and related rights in the work worldwide are waived through +the [CC0 1.0 Universal public domain +dedication](https://creativecommons.org/publicdomain/zero/1.0/). + +All contributions to this project will be released under the CC0 +dedication. By submitting a pull request, you are agreeing to comply +with this waiver of copyright interest. diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..670154e --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,116 @@ +CC0 1.0 Universal + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator and +subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for the +purpose of contributing to a commons of creative, cultural and scientific +works ("Commons") that the public can reliably and without fear of later +claims of infringement build upon, modify, incorporate in other works, reuse +and redistribute as freely as possible in any form whatsoever and for any +purposes, including without limitation commercial purposes. These owners may +contribute to the Commons to promote the ideal of a free culture and the +further production of creative, cultural and scientific works, or to gain +reputation or greater distribution for their Work in part through the use and +efforts of others. + +For these and/or other purposes and motivations, and without any expectation +of additional consideration or compensation, the person associating CC0 with a +Work (the "Affirmer"), to the extent that he or she is an owner of Copyright +and Related Rights in the Work, voluntarily elects to apply CC0 to the Work +and publicly distribute the Work under its terms, with knowledge of his or her +Copyright and Related Rights in the Work and the meaning and intended legal +effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not limited +to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, communicate, + and translate a Work; + + ii. moral rights retained by the original author(s) and/or performer(s); + + iii. publicity and privacy rights pertaining to a person's image or likeness + depicted in a Work; + + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + + v. rights protecting the extraction, dissemination, use and reuse of data in + a Work; + + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation thereof, + including any amended or successor version of such directive); and + + vii. other similar, equivalent or corresponding rights throughout the world + based on applicable law or treaty, and any national implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention of, +applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and +unconditionally waives, abandons, and surrenders all of Affirmer's Copyright +and Related Rights and associated claims and causes of action, whether now +known or unknown (including existing as well as future claims and causes of +action), in the Work (i) in all territories worldwide, (ii) for the maximum +duration provided by applicable law or treaty (including future time +extensions), (iii) in any current or future medium and for any number of +copies, and (iv) for any purpose whatsoever, including without limitation +commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes +the Waiver for the benefit of each member of the public at large and to the +detriment of Affirmer's heirs and successors, fully intending that such Waiver +shall not be subject to revocation, rescission, cancellation, termination, or +any other legal or equitable action to disrupt the quiet enjoyment of the Work +by the public as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason be +judged legally invalid or ineffective under applicable law, then the Waiver +shall be preserved to the maximum extent permitted taking into account +Affirmer's express Statement of Purpose. In addition, to the extent the Waiver +is so judged Affirmer hereby grants to each affected person a royalty-free, +non transferable, non sublicensable, non exclusive, irrevocable and +unconditional license to exercise Affirmer's Copyright and Related Rights in +the Work (i) in all territories worldwide, (ii) for the maximum duration +provided by applicable law or treaty (including future time extensions), (iii) +in any current or future medium and for any number of copies, and (iv) for any +purpose whatsoever, including without limitation commercial, advertising or +promotional purposes (the "License"). The License shall be deemed effective as +of the date CC0 was applied by Affirmer to the Work. Should any part of the +License for any reason be judged legally invalid or ineffective under +applicable law, such partial invalidity or ineffectiveness shall not +invalidate the remainder of the License, and in such case Affirmer hereby +affirms that he or she will not (i) exercise any of his or her remaining +Copyright and Related Rights in the Work or (ii) assert any associated claims +and causes of action with respect to the Work, in either case contrary to +Affirmer's express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + + b. Affirmer offers the Work as-is and makes no representations or warranties + of any kind concerning the Work, express, implied, statutory or otherwise, + including without limitation warranties of title, merchantability, fitness + for a particular purpose, non infringement, or the absence of latent or + other defects, accuracy, or the present or absence of errors, whether or not + discoverable, all to the greatest extent permissible under applicable law. + + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without limitation + any person's Copyright and Related Rights in the Work. Further, Affirmer + disclaims responsibility for obtaining any necessary consents, permissions + or other rights required for any use of the Work. + + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to this + CC0 or use of the Work. + +For more information, please see + diff --git a/README.md b/README.md new file mode 100644 index 0000000..5d5c4e3 --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +# generic-skeleton # + +[![Build Status](https://travis-ci.com/cisagov/generic-skeleton.svg?branch=develop)](https://travis-ci.com/cisagov/generic-skeleton) + +This is a generic skeleton project that can be used to quickly get a +new [cisagov](https://github.com/cisagov) GitHub project started. +This skeleton project contains [licensing information](LICENSE.md), as +well as [pre-commit hooks](https://pre-commit.com) and a [Travis +CI](https://travis-ci.com) configuration appropriate for the major +languages that we use. + +In many cases you will instead want to use one of the more specific +skeleton projects derived from this one. + +## Contributing ## + +We welcome contributions! Please see [here](CONTRIBUTING.md) for +details. + +## License ## + +This project is in the worldwide [public domain](LICENSE.md). + +This project is in the public domain within the United States, and +copyright and related rights in the work worldwide are waived through +the [CC0 1.0 Universal public domain +dedication](https://creativecommons.org/publicdomain/zero/1.0/). + +All contributions to this project will be released under the CC0 +dedication. By submitting a pull request, you are agreeing to comply +with this waiver of copyright interest. + +## Author Information ## + +Shane Frasier - diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..f122cc5 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements-test.txt +ipython diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..35170aa --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +black +pre-commit +pyupgrade From 40aec83a182b98a0c69af43267f52537cdab950f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 11 Mar 2019 09:00:27 -0400 Subject: [PATCH 002/253] Remove molecule test from Travis CI config --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 028a17e..a4493e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,4 +8,3 @@ install: - pip install --upgrade -r requirements-test.txt script: - pre-commit run --all-files - - molecule test From 389bc397deb9b3c4282d215839ba7b35af94639f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 11 Mar 2019 09:08:29 -0400 Subject: [PATCH 003/253] Rename repo to skeleton-generic --- CONTRIBUTING.md | 10 +++++----- README.md | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a7a2f33..ba1ed84 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Before contributing, we encourage you to read our CONTRIBUTING policy If you want to report a bug or request a new feature, the most direct method is to [create an -issue](https://github.com/cisagov/generic-skeleton/issues) in this +issue](https://github.com/cisagov/skeleton-generic/issues) in this repository. We recommend that you first search through existing issues (both open and closed) to check if your particular issue has already been reported. If it has then you might want to add a comment @@ -25,7 +25,7 @@ one. ## Pull requests ## If you choose to [submit a pull -request](https://github.com/cisagov/generic-skeleton/pulls), you will +request](https://github.com/cisagov/skeleton-generic/pulls), you will notice that our continuous integration (CI) system runs a fairly extensive set of linters and syntax checkers. Your pull request may fail these checks, and that's OK. If you want you can stop there and @@ -79,9 +79,9 @@ can create and configure the Python virtual environment with these commands: ```bash -cd generic-skeleton -pyenv virtualenv generic-skeleton -pyenv local generic-skeleton +cd skeleton-generic +pyenv virtualenv skeleton-generic +pyenv local skeleton-generic pip install -r requirements-dev.txt ``` diff --git a/README.md b/README.md index 5d5c4e3..930ea2a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# generic-skeleton # +# skeleton-generic # -[![Build Status](https://travis-ci.com/cisagov/generic-skeleton.svg?branch=develop)](https://travis-ci.com/cisagov/generic-skeleton) +[![Build Status](https://travis-ci.com/cisagov/skeleton-generic.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-generic) This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. From d6eb32fabfc83de7f8161d69be5409000f9491fe Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 11 Mar 2019 09:17:05 -0400 Subject: [PATCH 004/253] Remove author information from README --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 930ea2a..2337970 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,3 @@ dedication](https://creativecommons.org/publicdomain/zero/1.0/). All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. - -## Author Information ## - -Shane Frasier - From 4d87c35b50b03789ac36f7353c093b535fc9b6ae Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 14 Mar 2019 22:34:36 -0400 Subject: [PATCH 005/253] Update the pyenv installation instructions In particular, mention pyenv-installer. --- CONTRIBUTING.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ba1ed84..ede2f47 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,11 +56,10 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux you will need to find a package for your particular distro, -or follow the manual instructions -[here](https://github.com/pyenv/pyenv#installation) and -[here](https://github.com/pyenv/pyenv-virtualenv#installation). Then -you will need to add the same bit above to your profile. +For Linux (or on the Mac, if you don't want to use `brew`) you can use +[pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to +install the necessary tools. When you are finished you will need to +add the same two lines above to your profile. For a list of Python versions that are already installed and ready to use with `pyenv`, use the command `pyenv versions`. To see a list of From 13a180736a772991eb9d385f86b5d81b593ea4b6 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 30 Mar 2019 11:45:50 -0400 Subject: [PATCH 006/253] Add pre-commit hook for docker-compose linting --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d339352..2247035 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,6 +61,10 @@ repos: - id: terraform_fmt - id: terraform_validate_no_variables - id: terraform_docs + - repo: https://github.com/IamTheFij/docker-pre-commit + rev: v1.0.0 + hooks: + - id: docker-compose-check - repo: https://github.com/prettier/prettier rev: 1.16.2 hooks: From 74e145257cea8392dbc7a2d44153a67a01fc20f5 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 30 Mar 2019 12:47:36 -0400 Subject: [PATCH 007/253] Remove black and pyupgrade from requirements-test They are not needed there. Also run pre-commit autoupdate to pick up the latest version of hooks. --- .pre-commit-config.yaml | 4 ++-- requirements-test.txt | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2247035..e72a49f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.14.0 + rev: v0.14.1 hooks: - id: markdownlint # The LICENSE.md must match the license text exactly for @@ -47,7 +47,7 @@ repos: hooks: - id: pyupgrade - repo: https://github.com/ambv/black - rev: 18.9b0 + rev: 19.3b0 hooks: - id: black - repo: https://github.com/ansible/ansible-lint.git diff --git a/requirements-test.txt b/requirements-test.txt index 35170aa..416634f 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1 @@ -black pre-commit -pyupgrade From 953752e41a2bc55097facaf26e63fe8b2b1945f3 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 3 Apr 2019 22:51:50 -0400 Subject: [PATCH 008/253] Add bandit pre-commit hook Also ran pre-commit autoupdate, which picked up a newer version of the pyupgrade hook. --- .pre-commit-config.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e72a49f..926a491 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,9 +43,13 @@ repos: hooks: - id: flake8 - repo: https://github.com/asottile/pyupgrade - rev: v1.12.0 + rev: v1.14.0 hooks: - id: pyupgrade + - repo: https://github.com/PyCQA/bandit + rev: 2a1dbab + hooks: + - id: bandit - repo: https://github.com/ambv/black rev: 19.3b0 hooks: From c9f8b721473a60d27b5dded49e24cdf77ed86927 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 15 Apr 2019 11:43:55 -0400 Subject: [PATCH 009/253] Ignore W503 from flake8 This is a warning about line breaks before binary operators. It no longer agrees with PEP8. See, for example, here: https://github.com/ambv/black/issues/21. Guido agrees here: https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b. --- .flake8 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 33cdc3e..d937d88 100644 --- a/.flake8 +++ b/.flake8 @@ -15,4 +15,9 @@ select = C,E,F,W,B,B950 # Ignore flake8's default warning about maximum line length, which has # a hard stop at the configured value. Instead we use # flake8-bugbear's B950, which allows up to 10% overage. -ignore = E501 +# +# Also ignore flake8's warning about line breaks before binary +# operators. It no longer agrees with PEP8. See, for example, here: +# https://github.com/ambv/black/issues/21. Guido agrees here: +# https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b. +ignore = E501,W503 From 8e0b1d73868912f6736ae76af100b2d24b4f53b6 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 3 May 2019 14:30:56 -0400 Subject: [PATCH 010/253] Add pydoc warnings to flake8 config Also add flake8-docstrings as an additional dependency in the .pre-commit-config.yml file. --- .flake8 | 4 +++- .pre-commit-config.yaml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index d937d88..92ff826 100644 --- a/.flake8 +++ b/.flake8 @@ -3,6 +3,8 @@ max-line-length = 80 # Select (turn on) # * Complexity violations reported by mccabe (C) - # http://flake8.pycqa.org/en/latest/user/error-codes.html#error-violation-codes +# * Documentation conventions compliance reported by pydocstyle (D) - +# http://www.pydocstyle.org/en/stable/error_codes.html # * Default errors and warnings reported by pycodestyle (E and W) - # https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes # * Default errors reported by pyflakes (F) - @@ -11,7 +13,7 @@ max-line-length = 80 # https://github.com/PyCQA/flake8-bugbear#list-of-warnings # * The B950 flake8-bugbear opinionated warning - # https://github.com/PyCQA/flake8-bugbear#opinionated-warnings -select = C,E,F,W,B,B950 +select = C,D,E,F,W,B,B950 # Ignore flake8's default warning about maximum line length, which has # a hard stop at the configured value. Instead we use # flake8-bugbear's B950, which allows up to 10% overage. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 926a491..fb12c99 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,6 +42,8 @@ repos: rev: 3.7.7 hooks: - id: flake8 + additional_dependencies: + - flake8-docstrings - repo: https://github.com/asottile/pyupgrade rev: v1.14.0 hooks: From 6a64a512e40ba6a3cc0e6788fc67207f3e77ebd2 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 3 May 2019 16:49:35 -0400 Subject: [PATCH 011/253] Add a configuration file for Bandit, and configure pre-commit to use it. --- .bandit.yml | 12 ++++++++++++ .pre-commit-config.yaml | 2 ++ 2 files changed, 14 insertions(+) create mode 100644 .bandit.yml diff --git a/.bandit.yml b/.bandit.yml new file mode 100644 index 0000000..f122c97 --- /dev/null +++ b/.bandit.yml @@ -0,0 +1,12 @@ +# Configuration file for the Bandit python security scanner +# https://bandit.readthedocs.io/en/latest/config.html + +# Test are first included by `tests`, and then excluded by `skips`. +# If `tests` is empty, all tests are are considered included. + +tests: + #- B101 + #- B102 + +skips: + #- B101 # skip "assert used" check since assertions are required in pytests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb12c99..2a03327 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,6 +52,8 @@ repos: rev: 2a1dbab hooks: - id: bandit + args: + - --config=.bandit.yml - repo: https://github.com/ambv/black rev: 19.3b0 hooks: From 1a98e0ed062cc55719893bbbeafc3df40510b42d Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 3 May 2019 16:51:21 -0400 Subject: [PATCH 012/253] Improve english --- .bandit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bandit.yml b/.bandit.yml index f122c97..46bab94 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,7 +1,7 @@ # Configuration file for the Bandit python security scanner # https://bandit.readthedocs.io/en/latest/config.html -# Test are first included by `tests`, and then excluded by `skips`. +# Tests are first included by `tests`, and then excluded by `skips`. # If `tests` is empty, all tests are are considered included. tests: From 19afa70f67a10f14d76b9132f508849146b35656 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 6 May 2019 10:44:34 -0400 Subject: [PATCH 013/253] Run pre-commit autoupdate Also update the URL for black since it moved GitHub orgs from ambv to python. --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a03327..0bac8c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.1.0 + rev: v2.2.1 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.14.1 + rev: v0.15.0 hooks: - id: markdownlint # The LICENSE.md must match the license text exactly for @@ -45,7 +45,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.14.0 + rev: v1.16.3 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -54,7 +54,7 @@ repos: - id: bandit args: - --config=.bandit.yml - - repo: https://github.com/ambv/black + - repo: https://github.com/python/black rev: 19.3b0 hooks: - id: black @@ -74,6 +74,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.16.2 + rev: 1.17.0 hooks: - id: prettier From eb4fbd751b57db9c728e7b3065edbfde50633d97 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 6 May 2019 11:16:05 -0400 Subject: [PATCH 014/253] Make .bandit.yml pass yamllint --- .bandit.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 46bab94..8ba42d1 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,3 +1,4 @@ +--- # Configuration file for the Bandit python security scanner # https://bandit.readthedocs.io/en/latest/config.html @@ -5,8 +6,8 @@ # If `tests` is empty, all tests are are considered included. tests: - #- B101 - #- B102 +# - B101 +# - B102 skips: - #- B101 # skip "assert used" check since assertions are required in pytests +# - B101 # skip "assert used" check since assertions are required in pytests From a9ff3a336c0e9c9ee153b22a0de3bf5d07f5e462 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 26 May 2019 11:49:13 -0400 Subject: [PATCH 015/253] Enable TravisCI caching for pip and pre-commit plugins This should speed up TravisCI builds. --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index a4493e4..2c70e21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,15 @@ dist: xenial language: python python: 3.7 +# pre-commit hooks can use Docker, so we should go ahead and enable it services: docker +# Cache pip packages and pre-commit plugins to speed up builds +cache: + pip: true + directories: + - $HOME/.cache/pre-commit + install: - pip install --upgrade -r requirements-test.txt script: From 4e9280eb2baf8a72fa7abdf03898efb8ca82e2a1 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 26 May 2019 11:57:27 -0400 Subject: [PATCH 016/253] Update pre-commit plugins to latest versions Via pre-commit autoupdate. --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0bac8c8..b6a0939 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.2.1 + rev: v2.2.3 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.15.0 + rev: v0.16.0 hooks: - id: markdownlint # The LICENSE.md must match the license text exactly for @@ -35,7 +35,7 @@ repos: hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell - rev: 1.0.4 + rev: 1.0.5 hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 @@ -45,11 +45,11 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.16.3 + rev: v1.17.1 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit - rev: 2a1dbab + rev: 1.6.0 hooks: - id: bandit args: @@ -74,6 +74,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.17.0 + rev: 1.17.1 hooks: - id: prettier From 1a6b6b4af4c468668e2e2b1ce7eb5aa0df38560b Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 28 May 2019 14:39:17 -0400 Subject: [PATCH 017/253] update pre-commit hooks --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6a0939..69ebc03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,7 +64,7 @@ repos: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.11.0 + rev: v1.12.0 hooks: - id: terraform_fmt - id: terraform_validate_no_variables From 82236155ac2cc6d7235d8d0a40b04c2f34996e32 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 28 May 2019 18:04:22 -0400 Subject: [PATCH 018/253] Add initial postfix templates and configurations --- .pre-commit-config.yaml | 1 + Dockerfile | 26 ++++++++++++++++++++ docker-compose.yml | 35 +++++++++++++++++++++++++++ secrets/fullchain.pem | 31 ++++++++++++++++++++++++ secrets/privkey.pem | 52 ++++++++++++++++++++++++++++++++++++++++ src/docker-entrypoint.sh | 16 +++++++++++++ templates/main.cf | 32 +++++++++++++++++++++++++ templates/master.cf | 9 +++++++ 8 files changed, 202 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 secrets/fullchain.pem create mode 100644 secrets/privkey.pem create mode 100755 src/docker-entrypoint.sh create mode 100644 templates/main.cf create mode 100644 templates/master.cf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69ebc03..2ac7afe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,7 @@ repos: args: - --allow-missing-credentials - id: detect-private-key + exclude: secrets/privkey.pem - id: end-of-file-fixer exclude: files/(issue|motd) - id: mixed-line-ending diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7064af7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM debian:buster-slim +MAINTAINER Mark Feldhousen + +RUN apt-get update && \ +apt-get install --no-install-recommends -y \ +ca-certificates \ +gettext-base \ +opendkim \ +opendkim-tools \ +postfix \ +sasl2-bin \ +&& apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +USER root +WORKDIR /root + +RUN mv /etc/postfix/master.cf /etc/postfix/master.cf.orig + +COPY ./templates ./templates/ +COPY ./src/docker-entrypoint.sh . + +VOLUME ["/var/log", "/var/spool/postfix"] +EXPOSE 25/TCP 587/TCP + +ENTRYPOINT ["./docker-entrypoint.sh"] +CMD ["postfix", "-v", "start-fg"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..525e957 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +--- +version: "3.7" + +secrets: + fullchain_pem: + file: ./secrets/fullchain.pem + privkey_pem: + file: ./secrets/privkey.pem + + +services: + postfix: + build: + context: . + dockerfile: Dockerfile + image: postfix + init: true + restart: always + environment: + - PRIMARY_DOMAIN=example.com + - RELAY_IP= + ports: + - target: "25" + published: "1025" + protocol: tcp + mode: host + - target: "587" + published: "1587" + protocol: tcp + mode: host + secrets: + - source: fullchain_pem + target: fullchain.pem + - source: privkey_pem + target: privkey.pem diff --git a/secrets/fullchain.pem b/secrets/fullchain.pem new file mode 100644 index 0000000..ddc24ba --- /dev/null +++ b/secrets/fullchain.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFXTCCA0WgAwIBAgIJAPWv/2ssPwHVMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTkwNTI4MjAxNDM0WhcNMjAwNTI3MjAxNDM0WjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAvclFvQ6WAkQXpwNksjjojlvAKkqTnHJ8vHaM0C3yrSm+aMPH0/lzLTuT +pv5eaSBUUzi5f/VjBFslH7kAGct4m1MJUfxRYdP4uZXqnfkiMyT8x7z+k6SbD22U +6Xxa7yV+hwkbhNDFOmcCWawgrERvfkSdyp/l94u+TWg5v/LvmkmsFRixT+U5dl/g +vSdXbAvjdrn+x/IRVMFrEDTm5QNCHrx1lTQf4giFl7VU820HQiNT3Y3JD7v+JHLO +DUyYZAA6bpg5vYYMbhxfegZu/C7DkT3/ZtpRXEPv1mR+koS7nPAtZmJ8t76GR/eU +A5rYLv1P6a4KvUf0/uRPGHQZsj57lVRqyR3TDi37aDywvcdTBQZHe9fNyYYe7g6s +ToKzY/Z+KwVuI+KcamVH6QLLhHOZ7IFIdB5PrSCMLHlDRP22GWV5lbFjxP+9H/2P +B5QkoBuPS4vV/GNxMlGlnbCoVDdUluypnsl0pdpae36PiFPlkA3dzJ1OcOt803jR +E3HHBg4Mq9rO51NQfxx3LsnKPVvajVChcEie1UEH+DQLEVciRBIp9Jho4SMh9xS9 +VXFRXXrnEKkfLOXwkd3Vy4upvyQrxpEMmj1SHSsFqJ4xz1uo5akpgwzB5I14oe+k +OK/sLP+B4GryWp78cASQQ/0ldbvOWxBxm7OVSPQB6F6vHVKNExsCAwEAAaNQME4w +HQYDVR0OBBYEFOAh81Tup7BSdYBjAPGyd1GW5GJhMB8GA1UdIwQYMBaAFOAh81Tu +p7BSdYBjAPGyd1GW5GJhMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB +ALBpfqJWpTf58mv1n6HuSLI7n/FGA0wxoVPmvG9qBV0uBx0kbsuJVBUXfO+FE/3x +aiTiCwKweqdRhA2H7TnGs12D1Dweh3jd1gs+inClN7w4Ge9hIMiDcc9K45wgP/Sk +IIJYgLfctGfzc5EetqWQFq1GEPpNPNpfBMC7Z/KcgiZ8+RmoIACDlJ+EwSkDuXgf +045n45cF2xqfpeq/qKZDfWwg5+js/LN7abDRxX+JjH1i678Wx+SOlxsRK2plmmIL +c46vSErWoRddwxyZyP4TqBCTgV6ZcoZSWKFvHrn4c2YZHMuagsBaDIBg05jfxv72 +ewuKeWIC0/2PkbJ+W15X+/Ltgru7gIcidt8Xm+JnBjvz0bCiS2qVGxu0DxBmbmjv +CW3pvaNhc7NWxnTbJdpC/G2wH1RV06CS2WIWuBXu/AlkVlI/HHqYD4fWdETx9VjM +821dJY0oGBVqK+2/2d9Q1J1bfzs/J2kLaNocERog2RMapygHoNI0qtfsQIyEecGw +LoDcYltzKM+tPbWsYc1lI1rNo0v0/Y2TjQ6Jq+P4eZQ9gf6XgmdLg5nIDPkO4til +epYT2sgG6TAMyzf475BX92ect9KWD1efAavL+aSxwpngWBc7uqYvvbX8w1EePX7G +EoxN6uctuyBtDpzhdKxEEVBZ7NfU6X/91ZgVdlOR/rN4 +-----END CERTIFICATE----- diff --git a/secrets/privkey.pem b/secrets/privkey.pem new file mode 100644 index 0000000..52e5024 --- /dev/null +++ b/secrets/privkey.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC9yUW9DpYCRBen +A2SyOOiOW8AqSpOccny8dozQLfKtKb5ow8fT+XMtO5Om/l5pIFRTOLl/9WMEWyUf +uQAZy3ibUwlR/FFh0/i5leqd+SIzJPzHvP6TpJsPbZTpfFrvJX6HCRuE0MU6ZwJZ +rCCsRG9+RJ3Kn+X3i75NaDm/8u+aSawVGLFP5Tl2X+C9J1dsC+N2uf7H8hFUwWsQ +NOblA0IevHWVNB/iCIWXtVTzbQdCI1PdjckPu/4kcs4NTJhkADpumDm9hgxuHF96 +Bm78LsORPf9m2lFcQ+/WZH6ShLuc8C1mYny3voZH95QDmtgu/U/prgq9R/T+5E8Y +dBmyPnuVVGrJHdMOLftoPLC9x1MFBkd7183Jhh7uDqxOgrNj9n4rBW4j4pxqZUfp +AsuEc5nsgUh0Hk+tIIwseUNE/bYZZXmVsWPE/70f/Y8HlCSgG49Li9X8Y3EyUaWd +sKhUN1SW7KmeyXSl2lp7fo+IU+WQDd3MnU5w63zTeNETcccGDgyr2s7nU1B/HHcu +yco9W9qNUKFwSJ7VQQf4NAsRVyJEEin0mGjhIyH3FL1VcVFdeucQqR8s5fCR3dXL +i6m/JCvGkQyaPVIdKwWonjHPW6jlqSmDDMHkjXih76Q4r+ws/4HgavJanvxwBJBD +/SV1u85bEHGbs5VI9AHoXq8dUo0TGwIDAQABAoICAQCRaDhKVXaRXeJRT8RC2F81 +Uw60WFcoMn9nVd0lU07vZWBBnF7qBeE88rx54cIsAV0aNgfKBhRLLhoPaAqvuLk7 +KC+n5Q3lSiby6e3MAyk0zk3uKttR+3fiJi9FhMWXHL8Ibu3qoJm72Vhvo/WUhwp1 +T9UlfcUQGL1BSW2Vp2f0aiWyNC0F7bZM/8CMrCvK2ID6Yh7WypyEt3xz+lQ9enWa +XwInwrv6zlSsm33u08YP4klLImq952ccPempPtozJAmg2njCwIWdh5ePQoaeKKYm +Db4062gSrOqA9JYVZCTqZQoju6majhsL4KBC8sxXlDU58OLBivQmpn4DWlClxEGi +IbY/FIE6WEhOrdoGPzIjAcC3OYYTasIMBDLdA0tODmtv9Nvst30IGZc4Pm/QIJOk +EGJo4hqWbxiy4gisWxHwYeQ9/EEwrrc3FP94VscVkT8x0i22w5WMLtcrnCGpwzMg +E10+9v4ZUZ7cu9V+IeWQUkeuP3xhumI7RIDVRHpGC6TfEk/Q2gNdsPL2E8ng2Ytx +KMI3Pj5FuYi7enIR9AWdBVmVc2u7nzJMF/ODAwY6GmqHxni7PD97cnYwCy7Gxp/S +DZqiiD32RHwUwBm0AgdLhftkgqyTN/qo/Bhmj9ieO2CkuAvTYoXG0VMzxCb9wBG/ +7BJSGcbwtTJOJGK7LvrDAQKCAQEA6Q45teOKcmOSw5ne2cXzXuaXZ0OOCkjJ2ens +M89YmKXDVEZRbGoHVtftInUpr0H2UJ/N268Ogfzw62enZ40WIGwNALvp9PkLvdT0 +6LD/4MhcgZGQ5WDwqfqwkOanHdw9HJb752yEJ+3OG+fojmKkOs6OoQk1Ypxv5+5K +OuG/qtiKKpSLbG/nKAbPsPObArBxyfH9pV5F2E6vy38lYoDTURlA2BXHPoXu9M4c +/K2BMmO5zvGu5VOpAtnag5CWUwVvnX9DKDYs+k+exErluEj+U8GbKNQUTE+1p6fT +j4KKNVZBgnavOST3Xm/i4qVbccF/CwUc387HPdK5FU6kn3evewKCAQEA0HiEAytq +jzlBBHm892tojRzvpQa65fT7khsxETLhABvqeWZ2h9lE8TJTLC46N4cG1MC/hnWB +Q7XzKd7jAeht41Lp0mlDWv6eqKN4VyXSpAYzATcEO739eja7WNTgkYB91eDSyT+K +DVaElaXMjw/uX9tBnqaVyEe8JDqHw9E3Gl0MLWi89ztYptaWvKjt0+QqENBc6o+G +K/qzO+B4o9AyjyYkUYVA87tRrDk746LA5DbkpLQKPmQ3lb1hvVysJOnEdRabu5ly +mC0HR9n2UwcU98Op/EX3D4MuCUoFB/HQNMXq7oRMg+AcfsG0/ENcbiY6o0yRhxHu +ACgcjTi/QKAI4QKCAQBbgzB6EZ0diafpkpQFI0uLKjStYcN2mlpYbRhIx9RcLErk +3q++SGwVV7hP3X2+ycH0qqtk5fpmZHIdnZgIe0gC9yqr7R3TCa/onKSGcmonU8Wv +Qv+IcmZN+Jg4bbmVahO9FDRaDSxfmWtjXc7dijI+vTkYVstVq2PtyI3xTQ+8AEdQ +rP+KVu6HsxT+wMlPZwVnbNRSiRAX/d3dpFGDul4/7BCgSPzxuhm4mu6a8W5X4Pzn +G9O3TQCClBTPsIi2lN3dFEnEknFa4MTRAy/tCwyCyvUoNQ67YFlOOgJCydmHVBVp +Kz1mzPMta/XFVXTw2DAQnbNW1pU523K9wSG3VIHdAoIBACJTZbE76dzRWZJKFUJM +DjgGBrOOiyGoF/Azx/2D+iZRcmcw5t1xefeZCLbimbVg51AKuL6EBJfIktRXHdvH +kKh4k4WQzYVjHW65E+yNjsRxPN67V1ga7Wy9LFXxH1T16kJYNXzrmGif0U7usOLx +hZeE+6YK2ejTXvg8JvSoM0GFBqdHcq3muK8n8EP6MMbN79s648G/hiEhs3dte4/F +jT2i0yIVJd+7/TO1bNYLi2VIYJd6CaHCUKC4QSqz4qhlUXLSGSxnlMXXzDYZfoSn +St2M+yVNw+Nq/x6KcI+hUl4OJKPHZu3j7e01Kf7LfKGqa8dNqTyrSBwAfssGB/+1 +GiECggEAJD0KWTfJrSbgCkMfp1fNkwNExW2+neB+MI1eIR1sWsu8rz1a5d/NIdQq +pkoJp4FQUgRFEK+CzPWbKBDOxDVwpZ5o84JzxAEc78tL8/QIYwbtw5ZOiHNZ+wS6 +OYk6weY7rro7PwzqsTXcGdg/yxtphwguveSQM8y6McqBNZKqlN2fvXY8a4KZtt8O +RXBwpsqYulHpMGPh2MsMJBGEEII7Y2WKZG41oU1SGb5J2tBdGixW0buQnr6qwBgL +Ie8VV5kgbei97WK1lwvosn3HetBYSEE0GWMvjx93yoeozV8L/IF1rf7xss2BSqzF +UjgsHxWMDJWcER8NHXkE5DQORLtKCA== +-----END PRIVATE KEY----- diff --git a/src/docker-entrypoint.sh b/src/docker-entrypoint.sh new file mode 100755 index 0000000..6436d6f --- /dev/null +++ b/src/docker-entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# shellcheck disable=SC2016 + +set -e + +if [ "$1" = 'postfix' ]; then + + # generate confgurations using environment variables + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/main.cf > /etc/postfix/main.cf + cp /etc/postfix/master.cf.orig /etc/postfix/master.cf + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/master.cf >> /etc/postfix/master.cf + + exec "$@" +fi + +exec "$@" diff --git a/templates/main.cf b/templates/main.cf new file mode 100644 index 0000000..d69a9f1 --- /dev/null +++ b/templates/main.cf @@ -0,0 +1,32 @@ +smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) +biff = no +append_dot_mydomain = no +readme_directory = no +smtpd_tls_cert_file=/run/secrets/fullchain.pem +smtpd_tls_key_file=/run/secrets/privkey.pem +smtpd_tls_security_level = may +smtp_tls_security_level = may +smtpd_tls_session_cache_database = btree:$data_directory/smtpd_scache +smtp_tls_session_cache_database = btree:$data_directory/smtp_scache +smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination +myhostname = ${PRIMARY_DOMAIN} +alias_maps = hash:/etc/aliases +alias_database = hash:/etc/aliases +myorigin = ${PRIMARY_DOMAIN} +mydestination = ${PRIMARY_DOMAIN}, localhost.com, , localhost +relayhost = +mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 ${RELAY_IP} +mailbox_command = procmail -a "\$EXTENSION" +mailbox_size_limit = 0 +recipient_delimiter = + +inet_interfaces = all +inet_protocols = ipv4 +milter_default_action = accept +milter_protocol = 6 +smtpd_milters = inet:12301,inet:localhost:54321 +non_smtpd_milters = inet:12301,inet:localhost:54321 +disable_vrfy_command = yes +smtp_tls_note_starttls_offer = yes +always_bcc = mailarchive@${PRIMARY_DOMAIN} +maillog_file = /dev/stdout +compatibility_level = 2 diff --git a/templates/master.cf b/templates/master.cf new file mode 100644 index 0000000..92ed8a7 --- /dev/null +++ b/templates/master.cf @@ -0,0 +1,9 @@ +submission inet n - - - - smtpd + -o syslog_name=postfix/submission + -o smtpd_tls_wrappermode=no + -o smtpd_tls_security_level=may + -o smtpd_sasl_auth_enable=yes + -o smtpd_recipient_restrictions=permit_mynetworks,permit_sasl_authenticated,reject + -o milter_macro_daemon_name=ORIGINATING + -o smtpd_sasl_type=dovecot + -o smtpd_sasl_path=private/auth From 443de5b3e60df523d9c25dc011316e68f732e14e Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 17:35:41 -0400 Subject: [PATCH 019/253] Add password secret for mailarchive user --- secrets/mailarchive_passwd.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 secrets/mailarchive_passwd.txt diff --git a/secrets/mailarchive_passwd.txt b/secrets/mailarchive_passwd.txt new file mode 100644 index 0000000..323fae0 --- /dev/null +++ b/secrets/mailarchive_passwd.txt @@ -0,0 +1 @@ +foobar From 98498d729140e93c738495731c34585a201fa694 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 17:36:16 -0400 Subject: [PATCH 020/253] Sort postfix config and add proper environment variables --- templates/main.cf | 50 +++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/templates/main.cf b/templates/main.cf index d69a9f1..09d9cc4 100644 --- a/templates/main.cf +++ b/templates/main.cf @@ -1,32 +1,32 @@ -smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) -biff = no -append_dot_mydomain = no -readme_directory = no -smtpd_tls_cert_file=/run/secrets/fullchain.pem -smtpd_tls_key_file=/run/secrets/privkey.pem -smtpd_tls_security_level = may -smtp_tls_security_level = may -smtpd_tls_session_cache_database = btree:$data_directory/smtpd_scache -smtp_tls_session_cache_database = btree:$data_directory/smtp_scache -smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination -myhostname = ${PRIMARY_DOMAIN} -alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases -myorigin = ${PRIMARY_DOMAIN} -mydestination = ${PRIMARY_DOMAIN}, localhost.com, , localhost -relayhost = -mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 ${RELAY_IP} -mailbox_command = procmail -a "\$EXTENSION" -mailbox_size_limit = 0 -recipient_delimiter = + +alias_maps = hash:/etc/aliases +always_bcc = mailarchive@${PRIMARY_DOMAIN} +append_dot_mydomain = no +biff = no +compatibility_level = 2 +disable_vrfy_command = yes inet_interfaces = all inet_protocols = ipv4 +mailbox_command = procmail -a "\$EXTENSION" +mailbox_size_limit = 0 +maillog_file = /dev/stdout milter_default_action = accept milter_protocol = 6 -smtpd_milters = inet:12301,inet:localhost:54321 +mydestination = ${PRIMARY_DOMAIN}, localhost.com, , localhost +myhostname = ${PRIMARY_DOMAIN} +mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 ${RELAY_IP} +myorigin = ${PRIMARY_DOMAIN} non_smtpd_milters = inet:12301,inet:localhost:54321 -disable_vrfy_command = yes +readme_directory = no +recipient_delimiter = + +relayhost = smtp_tls_note_starttls_offer = yes -always_bcc = mailarchive@${PRIMARY_DOMAIN} -maillog_file = /dev/stdout -compatibility_level = 2 +smtp_tls_security_level = may +smtp_tls_session_cache_database = btree:$data_directory/smtp_scache +smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) +smtpd_milters = inet:12301,inet:localhost:54321 +smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination +smtpd_tls_cert_file=/run/secrets/fullchain.pem +smtpd_tls_key_file=/run/secrets/privkey.pem +smtpd_tls_security_level = may +smtpd_tls_session_cache_database = btree:$data_directory/smtpd_scache From baf31426f3a57ac266169ff3c0b28af3c5257fe0 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 17:36:38 -0400 Subject: [PATCH 021/253] Add configurations for supporting services --- templates/TrustedHosts | 4 ++++ templates/dovecot.conf | 38 ++++++++++++++++++++++++++++++++++++++ templates/opendkim.conf | 17 +++++++++++++++++ templates/opendmarc.conf | 10 ++++++++++ 4 files changed, 69 insertions(+) create mode 100644 templates/TrustedHosts create mode 100644 templates/dovecot.conf create mode 100644 templates/opendkim.conf create mode 100644 templates/opendmarc.conf diff --git a/templates/TrustedHosts b/templates/TrustedHosts new file mode 100644 index 0000000..53aaa67 --- /dev/null +++ b/templates/TrustedHosts @@ -0,0 +1,4 @@ +127.0.0.1 +localhost +${PRIMARY_DOMAIN} +${RELAY_IP} diff --git a/templates/dovecot.conf b/templates/dovecot.conf new file mode 100644 index 0000000..167b8a6 --- /dev/null +++ b/templates/dovecot.conf @@ -0,0 +1,38 @@ +disable_plaintext_auth = no +mail_privileged_group = mail +mail_location = mbox:~/mail:INBOX=/var/mail/%u +userdb { + driver = passwd +} +passdb { + args = %s + driver = pam +} +protocols = " imap" +protocol imap { + mail_plugins = " autocreate" +} +plugin { + autocreate = Trash + autocreate2 = Sent + autosubscribe = Trash + autosubscribe2 = Sent +} +service imap-login { + inet_listener imap { + port = 0 + } + inet_listener imaps { + port = 993 + } +} +service auth { + unix_listener /var/spool/postfix/private/auth { + group = postfix + mode = 0660 + user = postfix + } +} +ssl=required +ssl_cert = Date: Thu, 30 May 2019 18:00:39 -0400 Subject: [PATCH 022/253] Update docker components --- Dockerfile | 15 +++++++++++++-- docker-compose.yml | 21 ++++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7064af7..6dd598e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,25 +2,36 @@ FROM debian:buster-slim MAINTAINER Mark Feldhousen RUN apt-get update && \ -apt-get install --no-install-recommends -y \ +DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ +dovecot-imapd \ +dovecot-lmtpd \ gettext-base \ +mailutils \ opendkim \ opendkim-tools \ +opendmarc \ postfix \ +procmail \ sasl2-bin \ && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN adduser mailarchive --quiet --disabled-password \ +--shell /usr/sbin/nologin --gecos "Mail Archive" + USER root WORKDIR /root +# make backups of configurations. These are modified at startup. RUN mv /etc/postfix/master.cf /etc/postfix/master.cf.orig +RUN mv /etc/default/opendkim /etc/default/opendkim.orig +RUN mv /etc/default/opendmarc /etc/default/opendmarc.orig COPY ./templates ./templates/ COPY ./src/docker-entrypoint.sh . VOLUME ["/var/log", "/var/spool/postfix"] -EXPOSE 25/TCP 587/TCP +EXPOSE 25/TCP 587/TCP 993/TCP ENTRYPOINT ["./docker-entrypoint.sh"] CMD ["postfix", "-v", "start-fg"] diff --git a/docker-compose.yml b/docker-compose.yml index 525e957..1a50d6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,8 @@ secrets: file: ./secrets/fullchain.pem privkey_pem: file: ./secrets/privkey.pem + mailarchive_passwd_txt: + file: ./secrets/mailarchive_passwd.txt services: @@ -18,7 +20,10 @@ services: restart: always environment: - PRIMARY_DOMAIN=example.com - - RELAY_IP= + - RELAY_IP=172.16.202.1/32 + networks: + front: + ipv4_address: 172.16.202.2 ports: - target: "25" published: "1025" @@ -28,8 +33,22 @@ services: published: "1587" protocol: tcp mode: host + - target: "993" + published: "1993" + protocol: tcp + mode: host secrets: - source: fullchain_pem target: fullchain.pem - source: privkey_pem target: privkey.pem + - source: mailarchive_passwd_txt + target: mailarchive_passwd.txt + +networks: + front: + driver: bridge + ipam: + driver: default + config: + - subnet: 172.16.202.0/24 From a031cf06b5a65ecf61bf16373d92b13c1109790b Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 18:01:55 -0400 Subject: [PATCH 023/253] Add custom docker entrypoint --- src/docker-entrypoint.sh | 64 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/src/docker-entrypoint.sh b/src/docker-entrypoint.sh index 6436d6f..68bd74a 100755 --- a/src/docker-entrypoint.sh +++ b/src/docker-entrypoint.sh @@ -2,15 +2,69 @@ # shellcheck disable=SC2016 set -e +#set -x + +function generate_configs() { + # configure postfix + echo "Generating postfix configurations for ${PRIMARY_DOMAIN}" + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/main.cf > /etc/postfix/main.cf + cp /etc/postfix/master.cf.orig /etc/postfix/master.cf + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/master.cf >> /etc/postfix/master.cf + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/opendkim.conf > /etc/opendkim.conf + + # configure opendkim + echo "Generating opendkim configurations for ${PRIMARY_DOMAIN}" + mkdir -p "/etc/opendkim/keys/${PRIMARY_DOMAIN}" + opendkim-genkey --verbose --bits=1024 --selector=mail --directory="/etc/opendkim/keys/${PRIMARY_DOMAIN}" + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/TrustedHosts > /etc/opendkim/TrustedHosts + cp /etc/default/opendkim.orig /etc/default/opendkim + echo 'SOCKET="inet:12301"' >> /etc/default/opendkim + chown -R opendkim:opendkim /etc/opendkim + + # configure opendmarc + echo "Generating opendmarc configurations for ${PRIMARY_DOMAIN}" + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/opendmarc.conf > /etc/opendmarc.conf + mkdir "/etc/opendmarc/" + echo "localhost" > /etc/opendmarc/ignore.hosts + chown -R opendmarc:opendmarc /etc/opendmarc + cp /etc/default/opendmarc.orig /etc/default/opendmarc + echo 'SOCKET="inet:54321"' >> /etc/default/opendmarc + + # configure dovecot + echo "Generating dovecot configurations for ${PRIMARY_DOMAIN}" + envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/dovecot.conf > /etc/dovecot/dovecot.conf + + # create a file marking the configuration as completed for this domain + echo "All configurations generated for ${PRIMARY_DOMAIN}" +} if [ "$1" = 'postfix' ]; then + echo "Starting mail server with:" + echo " PRIMARY_DOMAIN=${PRIMARY_DOMAIN}" + echo " RELAY_IP=${RELAY_IP}" + + # check to see if the configuration was completed for this domain + if [[ ! -f conf_gen_done.txt ]] || [[ $(< conf_gen_done.txt) != "${PRIMARY_DOMAIN}" ]]; then + generate_configs + echo "${PRIMARY_DOMAIN}" > conf_gen_done.txt + else + echo "Configurations already generated for ${PRIMARY_DOMAIN}, preserving." + fi + + # set password of mailarchive user to secret + echo "mailarchive:$(< /run/secrets/mailarchive_passwd.txt)" | chpasswd + + # postfix needs fresh copies of files in its chroot jail + cp /etc/{hosts,localtime,nsswitch.conf,resolv.conf,services} /var/spool/postfix/etc/ - # generate confgurations using environment variables - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/main.cf > /etc/postfix/main.cf - cp /etc/postfix/master.cf.orig /etc/postfix/master.cf - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/master.cf >> /etc/postfix/master.cf + echo "--------------------------------------------" + cat "/etc/opendkim/keys/${PRIMARY_DOMAIN}/mail.txt" + echo "--------------------------------------------" - exec "$@" + opendmarc + opendkim + dovecot + exec "$@" fi exec "$@" From 35f407018ede943e97e38cd07116d849d2e03aed Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 18:52:56 -0400 Subject: [PATCH 024/253] Move templates and secrets into src --- .pre-commit-config.yaml | 2 +- Dockerfile | 2 +- docker-compose.yml | 6 +++--- {secrets => src/secrets}/fullchain.pem | 0 {secrets => src/secrets}/mailarchive_passwd.txt | 0 {secrets => src/secrets}/privkey.pem | 0 {templates => src/templates}/TrustedHosts | 0 {templates => src/templates}/dovecot.conf | 0 {templates => src/templates}/main.cf | 0 {templates => src/templates}/master.cf | 0 {templates => src/templates}/opendkim.conf | 0 {templates => src/templates}/opendmarc.conf | 0 12 files changed, 5 insertions(+), 5 deletions(-) rename {secrets => src/secrets}/fullchain.pem (100%) rename {secrets => src/secrets}/mailarchive_passwd.txt (100%) rename {secrets => src/secrets}/privkey.pem (100%) rename {templates => src/templates}/TrustedHosts (100%) rename {templates => src/templates}/dovecot.conf (100%) rename {templates => src/templates}/main.cf (100%) rename {templates => src/templates}/master.cf (100%) rename {templates => src/templates}/opendkim.conf (100%) rename {templates => src/templates}/opendmarc.conf (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ac7afe..189b5dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: args: - --allow-missing-credentials - id: detect-private-key - exclude: secrets/privkey.pem + exclude: src/secrets/privkey.pem - id: end-of-file-fixer exclude: files/(issue|motd) - id: mixed-line-ending diff --git a/Dockerfile b/Dockerfile index 6dd598e..a7982d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN mv /etc/postfix/master.cf /etc/postfix/master.cf.orig RUN mv /etc/default/opendkim /etc/default/opendkim.orig RUN mv /etc/default/opendmarc /etc/default/opendmarc.orig -COPY ./templates ./templates/ +COPY ./src/templates ./templates/ COPY ./src/docker-entrypoint.sh . VOLUME ["/var/log", "/var/spool/postfix"] diff --git a/docker-compose.yml b/docker-compose.yml index 1a50d6f..a2abb7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,11 +3,11 @@ version: "3.7" secrets: fullchain_pem: - file: ./secrets/fullchain.pem + file: ./src/secrets/fullchain.pem privkey_pem: - file: ./secrets/privkey.pem + file: ./src/secrets/privkey.pem mailarchive_passwd_txt: - file: ./secrets/mailarchive_passwd.txt + file: ./src/secrets/mailarchive_passwd.txt services: diff --git a/secrets/fullchain.pem b/src/secrets/fullchain.pem similarity index 100% rename from secrets/fullchain.pem rename to src/secrets/fullchain.pem diff --git a/secrets/mailarchive_passwd.txt b/src/secrets/mailarchive_passwd.txt similarity index 100% rename from secrets/mailarchive_passwd.txt rename to src/secrets/mailarchive_passwd.txt diff --git a/secrets/privkey.pem b/src/secrets/privkey.pem similarity index 100% rename from secrets/privkey.pem rename to src/secrets/privkey.pem diff --git a/templates/TrustedHosts b/src/templates/TrustedHosts similarity index 100% rename from templates/TrustedHosts rename to src/templates/TrustedHosts diff --git a/templates/dovecot.conf b/src/templates/dovecot.conf similarity index 100% rename from templates/dovecot.conf rename to src/templates/dovecot.conf diff --git a/templates/main.cf b/src/templates/main.cf similarity index 100% rename from templates/main.cf rename to src/templates/main.cf diff --git a/templates/master.cf b/src/templates/master.cf similarity index 100% rename from templates/master.cf rename to src/templates/master.cf diff --git a/templates/opendkim.conf b/src/templates/opendkim.conf similarity index 100% rename from templates/opendkim.conf rename to src/templates/opendkim.conf diff --git a/templates/opendmarc.conf b/src/templates/opendmarc.conf similarity index 100% rename from templates/opendmarc.conf rename to src/templates/opendmarc.conf From bf661acd28fd400614dd895fd0291025f7c03b84 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 18:53:20 -0400 Subject: [PATCH 025/253] Update URLs --- CONTRIBUTING.md | 10 +++++----- README.md | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ede2f47..59a01fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Before contributing, we encourage you to read our CONTRIBUTING policy If you want to report a bug or request a new feature, the most direct method is to [create an -issue](https://github.com/cisagov/skeleton-generic/issues) in this +issue](https://github.com/cisagov/docker-postfix/issues) in this repository. We recommend that you first search through existing issues (both open and closed) to check if your particular issue has already been reported. If it has then you might want to add a comment @@ -25,7 +25,7 @@ one. ## Pull requests ## If you choose to [submit a pull -request](https://github.com/cisagov/skeleton-generic/pulls), you will +request](https://github.com/cisagov/docker-postfix/pulls), you will notice that our continuous integration (CI) system runs a fairly extensive set of linters and syntax checkers. Your pull request may fail these checks, and that's OK. If you want you can stop there and @@ -78,9 +78,9 @@ can create and configure the Python virtual environment with these commands: ```bash -cd skeleton-generic -pyenv virtualenv skeleton-generic -pyenv local skeleton-generic +cd docker-postfix +pyenv virtualenv docker-postfix +pyenv local docker-postfix pip install -r requirements-dev.txt ``` diff --git a/README.md b/README.md index 2337970..7239e32 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# skeleton-generic # +# docker-postfix # -[![Build Status](https://travis-ci.com/cisagov/skeleton-generic.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-generic) +[![Build Status](https://travis-ci.com/cisagov/docker-postfix.svg?branch=develop)](https://travis-ci.com/cisagov/docker-postfix) This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. From b84b6b1b6d69004108b9cf04f5704dfc6e30346a Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 20:36:52 -0400 Subject: [PATCH 026/253] Get ready for docker hub integration --- .travis.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.travis.yml b/.travis.yml index 2c70e21..1bbdf65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,11 @@ python: 3.7 # pre-commit hooks can use Docker, so we should go ahead and enable it services: docker +env: + global: + - IMAGE_NAME=dhsncats/postfix + - DOCKER_USER=felddy + # Cache pip packages and pre-commit plugins to speed up builds cache: pip: true @@ -13,5 +18,17 @@ cache: install: - pip install --upgrade -r requirements-test.txt + script: - pre-commit run --all-files + #- travis_scripts/build_docker_image.sh + +after_success: + - coveralls + +# deploy: +# - provider: script +# script: bash travis_scripts/deploy_to_docker_hub.sh +# on: +# tags: true +# python: '3.7' From f7ba54655719794e0169f9547e41581e6029105f Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 20:37:17 -0400 Subject: [PATCH 027/253] Add scripts to support travis --- bump_version.sh | 47 ++++++++++++++++++++++++++ travis_scripts/build_docker_image.sh | 8 +++++ travis_scripts/deploy_to_docker_hub.sh | 9 +++++ 3 files changed, 64 insertions(+) create mode 100755 bump_version.sh create mode 100755 travis_scripts/build_docker_image.sh create mode 100755 travis_scripts/deploy_to_docker_hub.sh diff --git a/bump_version.sh b/bump_version.sh new file mode 100755 index 0000000..0de513d --- /dev/null +++ b/bump_version.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# bump_version.sh (show|major|minor|patch|prerelease|build) + +set -o nounset +set -o errexit +set -o pipefail + +VERSION_FILE=cyhy/mailer/__init__.py + +HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" + +old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) + +if [ $# -ne 1 ] +then + echo "$HELP_INFORMATION" +else + case $1 in + major|minor|patch|prerelease|build) + new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bumping version from $old_version to $new_version" + git push + ;; + finalize) + new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bumping version from $old_version to $new_version" + git push + ;; + show) + echo "$old_version" + ;; + *) + echo "$HELP_INFORMATION" + ;; + esac +fi diff --git a/travis_scripts/build_docker_image.sh b/travis_scripts/build_docker_image.sh new file mode 100755 index 0000000..862f220 --- /dev/null +++ b/travis_scripts/build_docker_image.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +version=$(./bump_version.sh show) +docker build -t "$IMAGE_NAME":"$version" . diff --git a/travis_scripts/deploy_to_docker_hub.sh b/travis_scripts/deploy_to_docker_hub.sh new file mode 100755 index 0000000..2be3e20 --- /dev/null +++ b/travis_scripts/deploy_to_docker_hub.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +echo "$DOCKER_PW" | docker login -u "$DOCKER_USER" --password-stdin +version=$(./bump_version.sh show) +docker push "$IMAGE_NAME":"$version" From b1f527b56598b31cdf8909d313e2568c11da62cd Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 20:40:45 -0400 Subject: [PATCH 028/253] Add container info to README --- README.md | 48 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7239e32..0ae8953 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,46 @@ -# docker-postfix # +# docker-postfix 🐳📮 # [![Build Status](https://travis-ci.com/cisagov/docker-postfix.svg?branch=develop)](https://travis-ci.com/cisagov/docker-postfix) -This is a generic skeleton project that can be used to quickly get a -new [cisagov](https://github.com/cisagov) GitHub project started. -This skeleton project contains [licensing information](LICENSE.md), as -well as [pre-commit hooks](https://pre-commit.com) and a [Travis -CI](https://travis-ci.com) configuration appropriate for the major -languages that we use. +Creates a Docker container with an installation of the +[postfix](http://postfix.org) MTA. Additionally it has an IMAP +server ([dovecot](https://dovecot.org)) for accessing the archvies +of sent email. All email is BCC's to the `mailarchive` account. -In many cases you will instead want to use one of the more specific -skeleton projects derived from this one. +## Usage ## + +A sample [docker composition](docker-compose.yml) is included in this repository. +To build and start the container use the command: `docker-compose up` + +### Ports ### + +By default this container will listen on the following ports: + +- 1025: `smtp` +- 1587: `submission` +- 1993: `imaps` + +### Environment Variables ### + +Two environment variables are used to generate the configurations at runtime: + +- `PRIMARY_DOMAIN`: the domain of the mail server +- `RELAY_IP`: (optional) an IP address that is allowed to relay mail without authentication + +### Secrets ### + +- `fullchain.pem`: public key +- `privkey.pem`: private key +- `mailarchive_password.txt`: password for the mailarchive user + +### Volumes ### + +Two optional volumes can be attached to this container to persist the +mail spool directory, as well as the logging directory. (Note that +the mail logs are available using the docker log command.) + +- `/var/spool/postfix`: mail queues +- `/var/log`: system logs ## Contributing ## From 61a2bafb7dd9465330a4c7e4bf34da3c7f1e6552 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 21:05:31 -0400 Subject: [PATCH 029/253] Remove uneeded --- requirements-dev.txt | 2 -- requirements-test.txt | 1 - 2 files changed, 3 deletions(-) delete mode 100644 requirements-dev.txt delete mode 100644 requirements-test.txt diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index f122cc5..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ --r requirements-test.txt -ipython diff --git a/requirements-test.txt b/requirements-test.txt deleted file mode 100644 index 416634f..0000000 --- a/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pre-commit From 2f4926b72ffc966af8beb3c8454caa827718c3be Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 21:05:48 -0400 Subject: [PATCH 030/253] Add a version file to bump --- bump_version.sh | 2 +- src/version.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 src/version.txt diff --git a/bump_version.sh b/bump_version.sh index 0de513d..7c23343 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -6,7 +6,7 @@ set -o nounset set -o errexit set -o pipefail -VERSION_FILE=cyhy/mailer/__init__.py +VERSION_FILE=src/version.txt HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" diff --git a/src/version.txt b/src/version.txt new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/src/version.txt @@ -0,0 +1 @@ +__version__ = "0.0.1" From ce21639ff226d88478d4abe72e833ab8e185b0ea Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 21:06:39 -0400 Subject: [PATCH 031/253] Add magic keys for travis deploy --- .travis.yml | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1bbdf65..82030d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,33 +2,38 @@ dist: xenial language: python python: 3.7 -# pre-commit hooks can use Docker, so we should go ahead and enable it services: docker - env: global: - IMAGE_NAME=dhsncats/postfix - DOCKER_USER=felddy - -# Cache pip packages and pre-commit plugins to speed up builds + - secure: >- + R8uSOMb+2aZV/bTK8EU3L5sYqNUiVZpS3kBU0XvVW3rnZgdW1QIHwLU2VOAv9kgpw1e + zXaAi3OPPboi3eMWTXGQGYM3prZePlpDk+cVwCZB5sQn4xw692p/VmNgnxrY9NTts0Y + 0qPSW9YdCEza8RLYbxaar8PGaLDdh5GwYBeBK8wrvd51PQa8J1IKDKRKfsbrKLRjHv8 + d8qSNT2yKa8vX63l80ftwHs07IOO2YKaUjtS4DlON+XHbLqXkfgVqQ8VUntW0qjb1M1 + KBGLckD6/qeCqMAPHDvuJY+fT2n+6Q0tDZbyJhYzYDN9eTTBuetIDoEZSU8vpvDslrG + qH8Pu93x4g63DkMl+lfMm6gNjkB8DLxOlHJvdjBHjiiIsSPdUdSxI/Y7GtkchD/Mkdq + JI5PyQSPnMvLAiNrhysGuIkKnD4hfDyR2RyePFw8zLgKj/Hsm71O1oX7I3aJu08xYL5 + EaXRFkeOMzCPamXNcKjpHWxP7UZqGROBe6e7eOKaJSFzpNZPczgHyi3XxoS8NNPIh0q + 4LTxeqHSUuzyEtg8az3meRiR9z+5LGA7mt2YZ7yyTPwizi2X2LVbx9yCtfnfEG/jrqI + tx6GFgOb7a02jEbDNQ5q0qR56IwFRSlw55EdWMo2gnh2PtJ044QaMzr/du2VM6wydAR + n2KsiL8piwAEc= cache: pip: true directories: - - $HOME/.cache/pre-commit - + - "$HOME/.cache/pre-commit" install: - pip install --upgrade -r requirements-test.txt - script: - pre-commit run --all-files - #- travis_scripts/build_docker_image.sh + - travis_scripts/build_docker_image.sh after_success: - coveralls - -# deploy: -# - provider: script -# script: bash travis_scripts/deploy_to_docker_hub.sh -# on: -# tags: true -# python: '3.7' +deploy: + - provider: script + script: bash travis_scripts/deploy_to_docker_hub.sh + on: + tags: true + python: '3.7' From 59c0c893aff55e08a010f7259cfde005b532683c Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 21:09:38 -0400 Subject: [PATCH 032/253] Remove install section --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82030d8..44e5f20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,6 @@ cache: pip: true directories: - "$HOME/.cache/pre-commit" -install: - - pip install --upgrade -r requirements-test.txt script: - pre-commit run --all-files - travis_scripts/build_docker_image.sh From 873d49c7146c7d54b78e84f7e7035cea7540e5a4 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 21:13:02 -0400 Subject: [PATCH 033/253] Re-enable pre-commit tests --- .travis.yml | 2 ++ requirements-test.txt | 1 + 2 files changed, 3 insertions(+) create mode 100644 requirements-test.txt diff --git a/.travis.yml b/.travis.yml index 44e5f20..82030d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ cache: pip: true directories: - "$HOME/.cache/pre-commit" +install: + - pip install --upgrade -r requirements-test.txt script: - pre-commit run --all-files - travis_scripts/build_docker_image.sh diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..416634f --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1 @@ +pre-commit From 752922cdb2ef2a001690f07cff73903652d6b95d Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 22:12:20 -0400 Subject: [PATCH 034/253] Update README --- README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0ae8953..312190a 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Creates a Docker container with an installation of the [postfix](http://postfix.org) MTA. Additionally it has an IMAP server ([dovecot](https://dovecot.org)) for accessing the archvies -of sent email. All email is BCC's to the `mailarchive` account. +of sent email. All email is BCC'd to the `mailarchive` account. ## Usage ## @@ -14,11 +14,14 @@ To build and start the container use the command: `docker-compose up` ### Ports ### -By default this container will listen on the following ports: +This container exposes the following ports: -- 1025: `smtp` -- 1587: `submission` -- 1993: `imaps` +- 25: `smtp` +- 587: `submission` +- 993: `imaps` + +The sample [docker composition](docker-compose.yml) publishes the +exposed ports at 1025, 1587, and 1993. ### Environment Variables ### From ea6ea18679c93c6bfcaf9d87900e7cc9fbe9ea74 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 May 2019 22:12:42 -0400 Subject: [PATCH 035/253] Add real image name to docker-compose build --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a2abb7d..c03837d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: build: context: . dockerfile: Dockerfile - image: postfix + image: dhsncats/postfix init: true restart: always environment: From 631f3a03a43752972e7361aafd8de21c8f4bea12 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 09:20:00 -0400 Subject: [PATCH 036/253] Move container build to install instead of script. This allows a hard fail of the container doesn't build. --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82030d8..6ffc591 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,10 +25,9 @@ cache: - "$HOME/.cache/pre-commit" install: - pip install --upgrade -r requirements-test.txt + - travis_scripts/build_docker_image.sh script: - pre-commit run --all-files - - travis_scripts/build_docker_image.sh - after_success: - coveralls deploy: From f3b72cf67589c0470f428e0acce6daf55949e86c Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 10:41:19 -0400 Subject: [PATCH 037/253] Limit bandit in tests tree --- .bandit.yml | 2 +- .pre-commit-config.yaml | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 8ba42d1..5ae161d 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -10,4 +10,4 @@ tests: # - B102 skips: -# - B101 # skip "assert used" check since assertions are required in pytests + - B101 # skip "assert used" check since assertions are required in pytests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 189b5dd..3b357eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,12 +49,22 @@ repos: rev: v1.17.1 hooks: - id: pyupgrade + # Run bandit on "tests" tree with a configuration - repo: https://github.com/PyCQA/bandit - rev: 1.6.0 + rev: 2a1dbab hooks: - id: bandit + name: bandit (tests tree) + files: tests args: - --config=.bandit.yml + # Run bandit everything but tests directory + - repo: https://github.com/PyCQA/bandit + rev: 2a1dbab + hooks: + - id: bandit + name: bandit (everything else) + exclude: tests - repo: https://github.com/python/black rev: 19.3b0 hooks: @@ -63,7 +73,6 @@ repos: rev: v4.1.0a0 hooks: - id: ansible-lint - # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: From f5c2107331b93b74b9bf29470bf008727cbbf334 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 10:41:46 -0400 Subject: [PATCH 038/253] Add container pytests --- .travis.yml | 8 +++- pytest.ini | 2 + requirements-test.txt | 1 + tests/container_test.py | 103 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 pytest.ini create mode 100644 tests/container_test.py diff --git a/.travis.yml b/.travis.yml index 6ffc591..bf01d1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,14 @@ cache: install: - pip install --upgrade -r requirements-test.txt - travis_scripts/build_docker_image.sh +before_script: + - docker-compose up -d + - docker-compose ps | grep -q postfix_postfix_1 script: - pre-commit run --all-files -after_success: - - coveralls + - pytest -v +after_script: + - docker-compose down deploy: - provider: script script: bash travis_scripts/deploy_to_docker_hub.sh diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..aada9d7 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = -v -ra diff --git a/requirements-test.txt b/requirements-test.txt index 416634f..51f1982 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1,2 @@ pre-commit +pytest diff --git a/tests/container_test.py b/tests/container_test.py new file mode 100644 index 0000000..59d1554 --- /dev/null +++ b/tests/container_test.py @@ -0,0 +1,103 @@ +#!/usr/bin/env pytest -vs +"""Tests for postfix container.""" + +from email.message import EmailMessage +from imaplib import IMAP4_SSL +import smtplib + +import pytest + +MESSAGE = """ +This is a test message sent during the unit tests. +""" +DOMAIN = "example.com" +TEST_USER = "mailarchive" +TEST_PW = "foobar" +IMAP_PORT = 1993 + + +@pytest.mark.parametrize("port", [1025, 1587]) +def test_sending_mail(port): + """Send an email message to the server.""" + msg = EmailMessage() + msg.set_content(MESSAGE) + msg["Subject"] = f"Test Message on port {port}" + msg["From"] = f"test@{DOMAIN}" + msg["To"] = f"mailarchive@{DOMAIN}" + with smtplib.SMTP("localhost", port=port) as s: + s.send_message(msg) + + +def test_imap_login(): + """Test logging in to the IMAP server.""" + with IMAP4_SSL("localhost", IMAP_PORT) as m: + m.login("mailarchive", "foobar") + + +def test_imap_messages_exist(): + """Test test existence of our test messages.""" + with IMAP4_SSL("localhost", IMAP_PORT) as m: + m.login(TEST_USER, TEST_PW) + typ, data = m.select() + assert typ == "OK", "Select did not return OK status" + message_count = int(data[0]) + print(f"inbox message count: {message_count}") + assert message_count > 0, "Expected message in the inbox" + + +def test_imap_access(): + """Test receiving message from the IMAP server.""" + with IMAP4_SSL("localhost", IMAP_PORT) as m: + m.login(TEST_USER, TEST_PW) + typ, data = m.select() + assert typ == "OK", "Select did not return OK status" + message_count = int(data[0]) + print(f"inbox message count: {message_count}") + typ, data = m.search(None, "ALL") + assert typ == "OK", "Search did not return OK status" + message_numbers = data[0].split() + for num in message_numbers: + typ, data = m.fetch(num, "(RFC822)") + assert typ == "OK", f"Fetch of message {num} did not return OK status" + print("-" * 40) + print(f"Message: {num}") + print(data[0][1].decode("utf-8")) + # mark messag as deleted + typ, data = m.store(num, "+FLAGS", "\\Deleted") + assert ( + typ == "OK" + ), f"Storing '\\deleted' flag on message {num} did not return OK status" + # expunge all deleted messages + typ, data = m.expunge() + assert typ == "OK", "Expunge did not return OK status" + + +def test_imap_delete_all(): + """Test deleting messages from the IMAP server.""" + with IMAP4_SSL("localhost", IMAP_PORT) as m: + m.login(TEST_USER, TEST_PW) + typ, data = m.select() + assert typ == "OK", "Select did not return OK status" + typ, data = m.search(None, "ALL") + assert typ == "OK", "Search did not return OK status" + message_numbers = data[0].split() + for num in message_numbers: + # mark messag as deleted + typ, data = m.store(num, "+FLAGS", "\\Deleted") + assert ( + typ == "OK" + ), f"Storing '\\deleted' flag on message {num} did not return OK status" + # expunge all deleted messages + typ, data = m.expunge() + assert typ == "OK", "Expunge did not return OK status" + + +def test_imap_messages_cleared(): + """Test that all messages were expunged.""" + with IMAP4_SSL("localhost", IMAP_PORT) as m: + m.login(TEST_USER, TEST_PW) + typ, data = m.select() + assert typ == "OK", "Select did not return OK status" + message_count = int(data[0]) + print(f"inbox message count: {message_count}") + assert message_count == 0, "Expected the inbox to be empty" From de8997ea4e7aaf3ac6895eb095e55a559d15310c Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 10:43:04 -0400 Subject: [PATCH 039/253] add ignore for pytest caches --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 073a081..d8e09a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__ .python-version +.pytest_cache From 4da142e28468090b461ec893b62771d06141108d Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 11:02:19 -0400 Subject: [PATCH 040/253] add latest tag to build for use by compose --- travis_scripts/build_docker_image.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/travis_scripts/build_docker_image.sh b/travis_scripts/build_docker_image.sh index 862f220..0a89fed 100755 --- a/travis_scripts/build_docker_image.sh +++ b/travis_scripts/build_docker_image.sh @@ -5,4 +5,5 @@ set -o errexit set -o pipefail version=$(./bump_version.sh show) -docker build -t "$IMAGE_NAME":"$version" . +# shellcheck disable=SC2140 +docker build -t "$IMAGE_NAME":"$version" -t "$IMAGE_NAME":"latest" . From 308fdb3d3a8af82e1af3cafa4d9387cffb5481b7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 11:03:33 -0400 Subject: [PATCH 041/253] Remove check that was failing due to race --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bf01d1b..c488104 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,6 @@ install: - travis_scripts/build_docker_image.sh before_script: - docker-compose up -d - - docker-compose ps | grep -q postfix_postfix_1 script: - pre-commit run --all-files - pytest -v From 72b108cc74902bf13af26f713517c88e9374c360 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 16:31:45 -0400 Subject: [PATCH 042/253] Add diceware for password generator --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index a7982d3..6a82707 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ MAINTAINER Mark Feldhousen RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ +diceware \ dovecot-imapd \ dovecot-lmtpd \ gettext-base \ From 4d3f10ea31c64c507ed7803c274bcf9bae40183f Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 16:32:29 -0400 Subject: [PATCH 043/253] Change secrets to new user.txt file --- docker-compose.yml | 8 ++++---- src/secrets/mailarchive_passwd.txt | 1 - src/secrets/users.txt | 10 ++++++++++ 3 files changed, 14 insertions(+), 5 deletions(-) delete mode 100644 src/secrets/mailarchive_passwd.txt create mode 100644 src/secrets/users.txt diff --git a/docker-compose.yml b/docker-compose.yml index c03837d..1f0d962 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,8 @@ secrets: file: ./src/secrets/fullchain.pem privkey_pem: file: ./src/secrets/privkey.pem - mailarchive_passwd_txt: - file: ./src/secrets/mailarchive_passwd.txt + users_txt: + file: ./src/secrets/users.txt services: @@ -42,8 +42,8 @@ services: target: fullchain.pem - source: privkey_pem target: privkey.pem - - source: mailarchive_passwd_txt - target: mailarchive_passwd.txt + - source: users_txt + target: users.txt networks: front: diff --git a/src/secrets/mailarchive_passwd.txt b/src/secrets/mailarchive_passwd.txt deleted file mode 100644 index 323fae0..0000000 --- a/src/secrets/mailarchive_passwd.txt +++ /dev/null @@ -1 +0,0 @@ -foobar diff --git a/src/secrets/users.txt b/src/secrets/users.txt new file mode 100644 index 0000000..86cd38b --- /dev/null +++ b/src/secrets/users.txt @@ -0,0 +1,10 @@ +# Define the users to be created at container startup. +# If is omitted for a user it will be generated and logged at startup +# username + +# The mailarchive user is mandatory since all mail is BCC'd to this user. +mailarchive foobar + +# define other users below as needed +testsender1 lemmy is god +testsender2 From 5234af1c945714950472ecbcad17efb41caf8321 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 16:35:23 -0400 Subject: [PATCH 044/253] Add dynamic user generator and tests. --- src/docker-entrypoint.sh | 24 +++++++++++++++-- tests/container_test.py | 58 ++++++++++++++++++++++++++++------------ 2 files changed, 63 insertions(+), 19 deletions(-) diff --git a/src/docker-entrypoint.sh b/src/docker-entrypoint.sh index 68bd74a..ce91e89 100755 --- a/src/docker-entrypoint.sh +++ b/src/docker-entrypoint.sh @@ -4,6 +4,7 @@ set -e #set -x + function generate_configs() { # configure postfix echo "Generating postfix configurations for ${PRIMARY_DOMAIN}" @@ -38,6 +39,24 @@ function generate_configs() { echo "All configurations generated for ${PRIMARY_DOMAIN}" } + +function generate_users() { + echo "Generating users and passwords:" + echo "--------------------------------------------" + while IFS=" " read -r username password || [ -n "$username" ] + do + if [ -z "$password" ]; then password=$(diceware -d-); + echo -e "$username\t$password" + else + echo -e "$username\t" + fi + adduser "$username" --quiet --disabled-password --shell /usr/sbin/nologin --gecos "" &>/dev/null || true + echo "$username:$password" | chpasswd || true + done + echo "--------------------------------------------" +} + + if [ "$1" = 'postfix' ]; then echo "Starting mail server with:" echo " PRIMARY_DOMAIN=${PRIMARY_DOMAIN}" @@ -51,12 +70,13 @@ if [ "$1" = 'postfix' ]; then echo "Configurations already generated for ${PRIMARY_DOMAIN}, preserving." fi - # set password of mailarchive user to secret - echo "mailarchive:$(< /run/secrets/mailarchive_passwd.txt)" | chpasswd + # generate the users from the secrets + grep -v '^#\|^$' /run/secrets/users.txt | generate_users # postfix needs fresh copies of files in its chroot jail cp /etc/{hosts,localtime,nsswitch.conf,resolv.conf,services} /var/spool/postfix/etc/ + echo "DKIM DNS entry:" echo "--------------------------------------------" cat "/etc/opendkim/keys/${PRIMARY_DOMAIN}/mail.txt" echo "--------------------------------------------" diff --git a/tests/container_test.py b/tests/container_test.py index 59d1554..0ccd9e9 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -11,44 +11,62 @@ This is a test message sent during the unit tests. """ DOMAIN = "example.com" -TEST_USER = "mailarchive" -TEST_PW = "foobar" +ARCHIVE_USER = "mailarchive" +ARCHIVE_PW = "foobar" +TEST_SEND_USER = "testsender1" +TEST_SEND_PW = "lemmy is god" IMAP_PORT = 1993 @pytest.mark.parametrize("port", [1025, 1587]) -def test_sending_mail(port): +@pytest.mark.parametrize("to_user", [ARCHIVE_USER, TEST_SEND_USER]) +def test_sending_mail(port, to_user): """Send an email message to the server.""" msg = EmailMessage() msg.set_content(MESSAGE) msg["Subject"] = f"Test Message on port {port}" msg["From"] = f"test@{DOMAIN}" - msg["To"] = f"mailarchive@{DOMAIN}" + msg["To"] = f"{to_user}@{DOMAIN}" with smtplib.SMTP("localhost", port=port) as s: s.send_message(msg) -def test_imap_login(): +@pytest.mark.parametrize( + "username,password", + [ + (ARCHIVE_USER, ARCHIVE_PW), + (TEST_SEND_USER, TEST_SEND_PW), + pytest.param(ARCHIVE_USER, TEST_SEND_PW, marks=pytest.mark.xfail), + pytest.param("your_mom", "so_fat", marks=pytest.mark.xfail), + ], +) +def test_imap_login(username, password): """Test logging in to the IMAP server.""" with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login("mailarchive", "foobar") + m.login(username, password) -def test_imap_messages_exist(): +@pytest.mark.parametrize( + "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] +) +def test_imap_messages_exist(username, password): """Test test existence of our test messages.""" with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(TEST_USER, TEST_PW) + m.login(username, password) typ, data = m.select() - assert typ == "OK", "Select did not return OK status" + assert typ == "OK", f"Select did not return OK status for {username}" message_count = int(data[0]) - print(f"inbox message count: {message_count}") - assert message_count > 0, "Expected message in the inbox" + print(f"{username} inbox message count: {message_count}") + assert message_count > 0, f"Expected message in the {username} inbox" -def test_imap_access(): +@pytest.mark.parametrize( + "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] +) +def test_imap_reading(username, password): """Test receiving message from the IMAP server.""" with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(TEST_USER, TEST_PW) + m.login(username, password) typ, data = m.select() assert typ == "OK", "Select did not return OK status" message_count = int(data[0]) @@ -72,10 +90,13 @@ def test_imap_access(): assert typ == "OK", "Expunge did not return OK status" -def test_imap_delete_all(): +@pytest.mark.parametrize( + "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] +) +def test_imap_delete_all(username, password): """Test deleting messages from the IMAP server.""" with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(TEST_USER, TEST_PW) + m.login(username, password) typ, data = m.select() assert typ == "OK", "Select did not return OK status" typ, data = m.search(None, "ALL") @@ -92,10 +113,13 @@ def test_imap_delete_all(): assert typ == "OK", "Expunge did not return OK status" -def test_imap_messages_cleared(): +@pytest.mark.parametrize( + "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] +) +def test_imap_messages_cleared(username, password): """Test that all messages were expunged.""" with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(TEST_USER, TEST_PW) + m.login(username, password) typ, data = m.select() assert typ == "OK", "Select did not return OK status" message_count = int(data[0]) From f25e910da6c2a3711a13bdf0f78a29a1ed7024e7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 16:39:40 -0400 Subject: [PATCH 045/253] Update documentation for user secrets --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 312190a..06cfa02 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Two environment variables are used to generate the configurations at runtime: - `fullchain.pem`: public key - `privkey.pem`: private key -- `mailarchive_password.txt`: password for the mailarchive user +- `users.txt`: account credentials to create at startup ### Volumes ### From 5d22bb478e0d020f929fee7a3f558af40dce93a5 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 31 May 2019 16:40:23 -0400 Subject: [PATCH 046/253] Bump version --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index f102a9c..3b93d0b 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.0.1" +__version__ = "0.0.2" From 38e4868e1aa3b1e52765c5337879baba702b5f4f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 3 Jun 2019 21:48:03 -0400 Subject: [PATCH 047/253] Add version slices to docker deploy --- .travis.yml | 16 ++++++++++++++-- travis_scripts/build_docker_image.sh | 9 --------- travis_scripts/deploy_to_docker_hub.sh | 9 --------- 3 files changed, 14 insertions(+), 20 deletions(-) delete mode 100755 travis_scripts/build_docker_image.sh delete mode 100755 travis_scripts/deploy_to_docker_hub.sh diff --git a/.travis.yml b/.travis.yml index c488104..8e952b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ cache: - "$HOME/.cache/pre-commit" install: - pip install --upgrade -r requirements-test.txt - - travis_scripts/build_docker_image.sh + - docker build -t "$IMAGE_NAME" . before_script: - docker-compose up -d script: @@ -33,9 +33,21 @@ script: - pytest -v after_script: - docker-compose down +before_deploy: + - version=$(./bump_version.sh show) + - IFS='.' read -r -a version_array <<< "$version" + - docker login -u "$DOCKER_USER" -p "$DOCKER_PW" + - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" + - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version}" + - docker tag "$IMAGE_NAME" + "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" + - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version_array[0]}" deploy: - provider: script - script: bash travis_scripts/deploy_to_docker_hub.sh + script: docker push "${IMAGE_NAME}:latest" && + docker push "${IMAGE_NAME}:${version}" && + docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" && + docker push "${IMAGE_NAME}:${version_array[0]}" on: tags: true python: '3.7' diff --git a/travis_scripts/build_docker_image.sh b/travis_scripts/build_docker_image.sh deleted file mode 100755 index 0a89fed..0000000 --- a/travis_scripts/build_docker_image.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o errexit -set -o pipefail - -version=$(./bump_version.sh show) -# shellcheck disable=SC2140 -docker build -t "$IMAGE_NAME":"$version" -t "$IMAGE_NAME":"latest" . diff --git a/travis_scripts/deploy_to_docker_hub.sh b/travis_scripts/deploy_to_docker_hub.sh deleted file mode 100755 index 2be3e20..0000000 --- a/travis_scripts/deploy_to_docker_hub.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o errexit -set -o pipefail - -echo "$DOCKER_PW" | docker login -u "$DOCKER_USER" --password-stdin -version=$(./bump_version.sh show) -docker push "$IMAGE_NAME":"$version" From 2c479d1434cab334d07e8b205746b5cef2748486 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 4 Jun 2019 14:00:22 -0400 Subject: [PATCH 048/253] Spell out flag names --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8e952b9..1f956d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,19 +24,19 @@ cache: directories: - "$HOME/.cache/pre-commit" install: - - pip install --upgrade -r requirements-test.txt - - docker build -t "$IMAGE_NAME" . + - pip install --upgrade --requirement requirements-test.txt + - docker build --tag "$IMAGE_NAME" . before_script: - - docker-compose up -d + - docker-compose up --detach script: - pre-commit run --all-files - - pytest -v + - pytest --verbose after_script: - docker-compose down before_deploy: - version=$(./bump_version.sh show) - IFS='.' read -r -a version_array <<< "$version" - - docker login -u "$DOCKER_USER" -p "$DOCKER_PW" + - docker login --username "$DOCKER_USER" --password "$DOCKER_PW" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version}" - docker tag "$IMAGE_NAME" From 06c04b94beda6bc253022c1dfc2d6ef3ab7f4074 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 4 Jun 2019 14:17:16 -0400 Subject: [PATCH 049/253] Rename repo --- CONTRIBUTING.md | 10 +++++----- README.md | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 59a01fc..f03cb56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Before contributing, we encourage you to read our CONTRIBUTING policy If you want to report a bug or request a new feature, the most direct method is to [create an -issue](https://github.com/cisagov/docker-postfix/issues) in this +issue](https://github.com/cisagov/postfix-docker/issues) in this repository. We recommend that you first search through existing issues (both open and closed) to check if your particular issue has already been reported. If it has then you might want to add a comment @@ -25,7 +25,7 @@ one. ## Pull requests ## If you choose to [submit a pull -request](https://github.com/cisagov/docker-postfix/pulls), you will +request](https://github.com/cisagov/postfix-docker/pulls), you will notice that our continuous integration (CI) system runs a fairly extensive set of linters and syntax checkers. Your pull request may fail these checks, and that's OK. If you want you can stop there and @@ -78,9 +78,9 @@ can create and configure the Python virtual environment with these commands: ```bash -cd docker-postfix -pyenv virtualenv docker-postfix -pyenv local docker-postfix +cd postfix-docker +pyenv virtualenv postfix-docker +pyenv local postfix-docker pip install -r requirements-dev.txt ``` diff --git a/README.md b/README.md index 06cfa02..20012dd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# docker-postfix 🐳📮 # +# postfix-docker 📮🐳 # -[![Build Status](https://travis-ci.com/cisagov/docker-postfix.svg?branch=develop)](https://travis-ci.com/cisagov/docker-postfix) +[![Build Status](https://travis-ci.com/cisagov/postfix-docker.svg?branch=develop)](https://travis-ci.com/cisagov/postfix-docker) Creates a Docker container with an installation of the [postfix](http://postfix.org) MTA. Additionally it has an IMAP From b36365282560d5f0787136f48a338ad1a3a0493b Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 4 Jun 2019 15:00:13 -0400 Subject: [PATCH 050/253] Spell out flag names --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index aada9d7..d302749 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -addopts = -v -ra +addopts = --verbose -ra From 121f27c2f048e07a581d3e4651655a4b89f4ba51 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 10:49:22 -0400 Subject: [PATCH 051/253] Remove non-skel files --- src/docker-entrypoint.sh | 90 ------------------------------------ src/secrets/fullchain.pem | 31 ------------- src/secrets/privkey.pem | 52 --------------------- src/secrets/users.txt | 10 ---- src/templates/TrustedHosts | 4 -- src/templates/dovecot.conf | 38 --------------- src/templates/main.cf | 32 ------------- src/templates/master.cf | 9 ---- src/templates/opendkim.conf | 17 ------- src/templates/opendmarc.conf | 10 ---- 10 files changed, 293 deletions(-) delete mode 100755 src/docker-entrypoint.sh delete mode 100644 src/secrets/fullchain.pem delete mode 100644 src/secrets/privkey.pem delete mode 100644 src/secrets/users.txt delete mode 100644 src/templates/TrustedHosts delete mode 100644 src/templates/dovecot.conf delete mode 100644 src/templates/main.cf delete mode 100644 src/templates/master.cf delete mode 100644 src/templates/opendkim.conf delete mode 100644 src/templates/opendmarc.conf diff --git a/src/docker-entrypoint.sh b/src/docker-entrypoint.sh deleted file mode 100755 index ce91e89..0000000 --- a/src/docker-entrypoint.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2016 - -set -e -#set -x - - -function generate_configs() { - # configure postfix - echo "Generating postfix configurations for ${PRIMARY_DOMAIN}" - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/main.cf > /etc/postfix/main.cf - cp /etc/postfix/master.cf.orig /etc/postfix/master.cf - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/master.cf >> /etc/postfix/master.cf - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/opendkim.conf > /etc/opendkim.conf - - # configure opendkim - echo "Generating opendkim configurations for ${PRIMARY_DOMAIN}" - mkdir -p "/etc/opendkim/keys/${PRIMARY_DOMAIN}" - opendkim-genkey --verbose --bits=1024 --selector=mail --directory="/etc/opendkim/keys/${PRIMARY_DOMAIN}" - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/TrustedHosts > /etc/opendkim/TrustedHosts - cp /etc/default/opendkim.orig /etc/default/opendkim - echo 'SOCKET="inet:12301"' >> /etc/default/opendkim - chown -R opendkim:opendkim /etc/opendkim - - # configure opendmarc - echo "Generating opendmarc configurations for ${PRIMARY_DOMAIN}" - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/opendmarc.conf > /etc/opendmarc.conf - mkdir "/etc/opendmarc/" - echo "localhost" > /etc/opendmarc/ignore.hosts - chown -R opendmarc:opendmarc /etc/opendmarc - cp /etc/default/opendmarc.orig /etc/default/opendmarc - echo 'SOCKET="inet:54321"' >> /etc/default/opendmarc - - # configure dovecot - echo "Generating dovecot configurations for ${PRIMARY_DOMAIN}" - envsubst '\$PRIMARY_DOMAIN \$RELAY_IP' < templates/dovecot.conf > /etc/dovecot/dovecot.conf - - # create a file marking the configuration as completed for this domain - echo "All configurations generated for ${PRIMARY_DOMAIN}" -} - - -function generate_users() { - echo "Generating users and passwords:" - echo "--------------------------------------------" - while IFS=" " read -r username password || [ -n "$username" ] - do - if [ -z "$password" ]; then password=$(diceware -d-); - echo -e "$username\t$password" - else - echo -e "$username\t" - fi - adduser "$username" --quiet --disabled-password --shell /usr/sbin/nologin --gecos "" &>/dev/null || true - echo "$username:$password" | chpasswd || true - done - echo "--------------------------------------------" -} - - -if [ "$1" = 'postfix' ]; then - echo "Starting mail server with:" - echo " PRIMARY_DOMAIN=${PRIMARY_DOMAIN}" - echo " RELAY_IP=${RELAY_IP}" - - # check to see if the configuration was completed for this domain - if [[ ! -f conf_gen_done.txt ]] || [[ $(< conf_gen_done.txt) != "${PRIMARY_DOMAIN}" ]]; then - generate_configs - echo "${PRIMARY_DOMAIN}" > conf_gen_done.txt - else - echo "Configurations already generated for ${PRIMARY_DOMAIN}, preserving." - fi - - # generate the users from the secrets - grep -v '^#\|^$' /run/secrets/users.txt | generate_users - - # postfix needs fresh copies of files in its chroot jail - cp /etc/{hosts,localtime,nsswitch.conf,resolv.conf,services} /var/spool/postfix/etc/ - - echo "DKIM DNS entry:" - echo "--------------------------------------------" - cat "/etc/opendkim/keys/${PRIMARY_DOMAIN}/mail.txt" - echo "--------------------------------------------" - - opendmarc - opendkim - dovecot - exec "$@" -fi - -exec "$@" diff --git a/src/secrets/fullchain.pem b/src/secrets/fullchain.pem deleted file mode 100644 index ddc24ba..0000000 --- a/src/secrets/fullchain.pem +++ /dev/null @@ -1,31 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFXTCCA0WgAwIBAgIJAPWv/2ssPwHVMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV -BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX -aWRnaXRzIFB0eSBMdGQwHhcNMTkwNTI4MjAxNDM0WhcNMjAwNTI3MjAxNDM0WjBF -MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 -ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC -CgKCAgEAvclFvQ6WAkQXpwNksjjojlvAKkqTnHJ8vHaM0C3yrSm+aMPH0/lzLTuT -pv5eaSBUUzi5f/VjBFslH7kAGct4m1MJUfxRYdP4uZXqnfkiMyT8x7z+k6SbD22U -6Xxa7yV+hwkbhNDFOmcCWawgrERvfkSdyp/l94u+TWg5v/LvmkmsFRixT+U5dl/g -vSdXbAvjdrn+x/IRVMFrEDTm5QNCHrx1lTQf4giFl7VU820HQiNT3Y3JD7v+JHLO -DUyYZAA6bpg5vYYMbhxfegZu/C7DkT3/ZtpRXEPv1mR+koS7nPAtZmJ8t76GR/eU -A5rYLv1P6a4KvUf0/uRPGHQZsj57lVRqyR3TDi37aDywvcdTBQZHe9fNyYYe7g6s -ToKzY/Z+KwVuI+KcamVH6QLLhHOZ7IFIdB5PrSCMLHlDRP22GWV5lbFjxP+9H/2P -B5QkoBuPS4vV/GNxMlGlnbCoVDdUluypnsl0pdpae36PiFPlkA3dzJ1OcOt803jR -E3HHBg4Mq9rO51NQfxx3LsnKPVvajVChcEie1UEH+DQLEVciRBIp9Jho4SMh9xS9 -VXFRXXrnEKkfLOXwkd3Vy4upvyQrxpEMmj1SHSsFqJ4xz1uo5akpgwzB5I14oe+k -OK/sLP+B4GryWp78cASQQ/0ldbvOWxBxm7OVSPQB6F6vHVKNExsCAwEAAaNQME4w -HQYDVR0OBBYEFOAh81Tup7BSdYBjAPGyd1GW5GJhMB8GA1UdIwQYMBaAFOAh81Tu -p7BSdYBjAPGyd1GW5GJhMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB -ALBpfqJWpTf58mv1n6HuSLI7n/FGA0wxoVPmvG9qBV0uBx0kbsuJVBUXfO+FE/3x -aiTiCwKweqdRhA2H7TnGs12D1Dweh3jd1gs+inClN7w4Ge9hIMiDcc9K45wgP/Sk -IIJYgLfctGfzc5EetqWQFq1GEPpNPNpfBMC7Z/KcgiZ8+RmoIACDlJ+EwSkDuXgf -045n45cF2xqfpeq/qKZDfWwg5+js/LN7abDRxX+JjH1i678Wx+SOlxsRK2plmmIL -c46vSErWoRddwxyZyP4TqBCTgV6ZcoZSWKFvHrn4c2YZHMuagsBaDIBg05jfxv72 -ewuKeWIC0/2PkbJ+W15X+/Ltgru7gIcidt8Xm+JnBjvz0bCiS2qVGxu0DxBmbmjv -CW3pvaNhc7NWxnTbJdpC/G2wH1RV06CS2WIWuBXu/AlkVlI/HHqYD4fWdETx9VjM -821dJY0oGBVqK+2/2d9Q1J1bfzs/J2kLaNocERog2RMapygHoNI0qtfsQIyEecGw -LoDcYltzKM+tPbWsYc1lI1rNo0v0/Y2TjQ6Jq+P4eZQ9gf6XgmdLg5nIDPkO4til -epYT2sgG6TAMyzf475BX92ect9KWD1efAavL+aSxwpngWBc7uqYvvbX8w1EePX7G -EoxN6uctuyBtDpzhdKxEEVBZ7NfU6X/91ZgVdlOR/rN4 ------END CERTIFICATE----- diff --git a/src/secrets/privkey.pem b/src/secrets/privkey.pem deleted file mode 100644 index 52e5024..0000000 --- a/src/secrets/privkey.pem +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC9yUW9DpYCRBen -A2SyOOiOW8AqSpOccny8dozQLfKtKb5ow8fT+XMtO5Om/l5pIFRTOLl/9WMEWyUf -uQAZy3ibUwlR/FFh0/i5leqd+SIzJPzHvP6TpJsPbZTpfFrvJX6HCRuE0MU6ZwJZ -rCCsRG9+RJ3Kn+X3i75NaDm/8u+aSawVGLFP5Tl2X+C9J1dsC+N2uf7H8hFUwWsQ -NOblA0IevHWVNB/iCIWXtVTzbQdCI1PdjckPu/4kcs4NTJhkADpumDm9hgxuHF96 -Bm78LsORPf9m2lFcQ+/WZH6ShLuc8C1mYny3voZH95QDmtgu/U/prgq9R/T+5E8Y -dBmyPnuVVGrJHdMOLftoPLC9x1MFBkd7183Jhh7uDqxOgrNj9n4rBW4j4pxqZUfp -AsuEc5nsgUh0Hk+tIIwseUNE/bYZZXmVsWPE/70f/Y8HlCSgG49Li9X8Y3EyUaWd -sKhUN1SW7KmeyXSl2lp7fo+IU+WQDd3MnU5w63zTeNETcccGDgyr2s7nU1B/HHcu -yco9W9qNUKFwSJ7VQQf4NAsRVyJEEin0mGjhIyH3FL1VcVFdeucQqR8s5fCR3dXL -i6m/JCvGkQyaPVIdKwWonjHPW6jlqSmDDMHkjXih76Q4r+ws/4HgavJanvxwBJBD -/SV1u85bEHGbs5VI9AHoXq8dUo0TGwIDAQABAoICAQCRaDhKVXaRXeJRT8RC2F81 -Uw60WFcoMn9nVd0lU07vZWBBnF7qBeE88rx54cIsAV0aNgfKBhRLLhoPaAqvuLk7 -KC+n5Q3lSiby6e3MAyk0zk3uKttR+3fiJi9FhMWXHL8Ibu3qoJm72Vhvo/WUhwp1 -T9UlfcUQGL1BSW2Vp2f0aiWyNC0F7bZM/8CMrCvK2ID6Yh7WypyEt3xz+lQ9enWa -XwInwrv6zlSsm33u08YP4klLImq952ccPempPtozJAmg2njCwIWdh5ePQoaeKKYm -Db4062gSrOqA9JYVZCTqZQoju6majhsL4KBC8sxXlDU58OLBivQmpn4DWlClxEGi -IbY/FIE6WEhOrdoGPzIjAcC3OYYTasIMBDLdA0tODmtv9Nvst30IGZc4Pm/QIJOk -EGJo4hqWbxiy4gisWxHwYeQ9/EEwrrc3FP94VscVkT8x0i22w5WMLtcrnCGpwzMg -E10+9v4ZUZ7cu9V+IeWQUkeuP3xhumI7RIDVRHpGC6TfEk/Q2gNdsPL2E8ng2Ytx -KMI3Pj5FuYi7enIR9AWdBVmVc2u7nzJMF/ODAwY6GmqHxni7PD97cnYwCy7Gxp/S -DZqiiD32RHwUwBm0AgdLhftkgqyTN/qo/Bhmj9ieO2CkuAvTYoXG0VMzxCb9wBG/ -7BJSGcbwtTJOJGK7LvrDAQKCAQEA6Q45teOKcmOSw5ne2cXzXuaXZ0OOCkjJ2ens -M89YmKXDVEZRbGoHVtftInUpr0H2UJ/N268Ogfzw62enZ40WIGwNALvp9PkLvdT0 -6LD/4MhcgZGQ5WDwqfqwkOanHdw9HJb752yEJ+3OG+fojmKkOs6OoQk1Ypxv5+5K -OuG/qtiKKpSLbG/nKAbPsPObArBxyfH9pV5F2E6vy38lYoDTURlA2BXHPoXu9M4c -/K2BMmO5zvGu5VOpAtnag5CWUwVvnX9DKDYs+k+exErluEj+U8GbKNQUTE+1p6fT -j4KKNVZBgnavOST3Xm/i4qVbccF/CwUc387HPdK5FU6kn3evewKCAQEA0HiEAytq -jzlBBHm892tojRzvpQa65fT7khsxETLhABvqeWZ2h9lE8TJTLC46N4cG1MC/hnWB -Q7XzKd7jAeht41Lp0mlDWv6eqKN4VyXSpAYzATcEO739eja7WNTgkYB91eDSyT+K -DVaElaXMjw/uX9tBnqaVyEe8JDqHw9E3Gl0MLWi89ztYptaWvKjt0+QqENBc6o+G -K/qzO+B4o9AyjyYkUYVA87tRrDk746LA5DbkpLQKPmQ3lb1hvVysJOnEdRabu5ly -mC0HR9n2UwcU98Op/EX3D4MuCUoFB/HQNMXq7oRMg+AcfsG0/ENcbiY6o0yRhxHu -ACgcjTi/QKAI4QKCAQBbgzB6EZ0diafpkpQFI0uLKjStYcN2mlpYbRhIx9RcLErk -3q++SGwVV7hP3X2+ycH0qqtk5fpmZHIdnZgIe0gC9yqr7R3TCa/onKSGcmonU8Wv -Qv+IcmZN+Jg4bbmVahO9FDRaDSxfmWtjXc7dijI+vTkYVstVq2PtyI3xTQ+8AEdQ -rP+KVu6HsxT+wMlPZwVnbNRSiRAX/d3dpFGDul4/7BCgSPzxuhm4mu6a8W5X4Pzn -G9O3TQCClBTPsIi2lN3dFEnEknFa4MTRAy/tCwyCyvUoNQ67YFlOOgJCydmHVBVp -Kz1mzPMta/XFVXTw2DAQnbNW1pU523K9wSG3VIHdAoIBACJTZbE76dzRWZJKFUJM -DjgGBrOOiyGoF/Azx/2D+iZRcmcw5t1xefeZCLbimbVg51AKuL6EBJfIktRXHdvH -kKh4k4WQzYVjHW65E+yNjsRxPN67V1ga7Wy9LFXxH1T16kJYNXzrmGif0U7usOLx -hZeE+6YK2ejTXvg8JvSoM0GFBqdHcq3muK8n8EP6MMbN79s648G/hiEhs3dte4/F -jT2i0yIVJd+7/TO1bNYLi2VIYJd6CaHCUKC4QSqz4qhlUXLSGSxnlMXXzDYZfoSn -St2M+yVNw+Nq/x6KcI+hUl4OJKPHZu3j7e01Kf7LfKGqa8dNqTyrSBwAfssGB/+1 -GiECggEAJD0KWTfJrSbgCkMfp1fNkwNExW2+neB+MI1eIR1sWsu8rz1a5d/NIdQq -pkoJp4FQUgRFEK+CzPWbKBDOxDVwpZ5o84JzxAEc78tL8/QIYwbtw5ZOiHNZ+wS6 -OYk6weY7rro7PwzqsTXcGdg/yxtphwguveSQM8y6McqBNZKqlN2fvXY8a4KZtt8O -RXBwpsqYulHpMGPh2MsMJBGEEII7Y2WKZG41oU1SGb5J2tBdGixW0buQnr6qwBgL -Ie8VV5kgbei97WK1lwvosn3HetBYSEE0GWMvjx93yoeozV8L/IF1rf7xss2BSqzF -UjgsHxWMDJWcER8NHXkE5DQORLtKCA== ------END PRIVATE KEY----- diff --git a/src/secrets/users.txt b/src/secrets/users.txt deleted file mode 100644 index 86cd38b..0000000 --- a/src/secrets/users.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Define the users to be created at container startup. -# If is omitted for a user it will be generated and logged at startup -# username - -# The mailarchive user is mandatory since all mail is BCC'd to this user. -mailarchive foobar - -# define other users below as needed -testsender1 lemmy is god -testsender2 diff --git a/src/templates/TrustedHosts b/src/templates/TrustedHosts deleted file mode 100644 index 53aaa67..0000000 --- a/src/templates/TrustedHosts +++ /dev/null @@ -1,4 +0,0 @@ -127.0.0.1 -localhost -${PRIMARY_DOMAIN} -${RELAY_IP} diff --git a/src/templates/dovecot.conf b/src/templates/dovecot.conf deleted file mode 100644 index 167b8a6..0000000 --- a/src/templates/dovecot.conf +++ /dev/null @@ -1,38 +0,0 @@ -disable_plaintext_auth = no -mail_privileged_group = mail -mail_location = mbox:~/mail:INBOX=/var/mail/%u -userdb { - driver = passwd -} -passdb { - args = %s - driver = pam -} -protocols = " imap" -protocol imap { - mail_plugins = " autocreate" -} -plugin { - autocreate = Trash - autocreate2 = Sent - autosubscribe = Trash - autosubscribe2 = Sent -} -service imap-login { - inet_listener imap { - port = 0 - } - inet_listener imaps { - port = 993 - } -} -service auth { - unix_listener /var/spool/postfix/private/auth { - group = postfix - mode = 0660 - user = postfix - } -} -ssl=required -ssl_cert = Date: Thu, 6 Jun 2019 10:49:32 -0400 Subject: [PATCH 052/253] revert version --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index 3b93d0b..f102a9c 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.0.2" +__version__ = "0.0.1" From bb0911a323f43938d590b149af93838781e1257a Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 10:49:52 -0400 Subject: [PATCH 053/253] Update project name --- CONTRIBUTING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f03cb56..958c559 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ Before contributing, we encourage you to read our CONTRIBUTING policy If you want to report a bug or request a new feature, the most direct method is to [create an -issue](https://github.com/cisagov/postfix-docker/issues) in this +issue](https://github.com/cisagov/skeleton-docker/issues) in this repository. We recommend that you first search through existing issues (both open and closed) to check if your particular issue has already been reported. If it has then you might want to add a comment @@ -25,7 +25,7 @@ one. ## Pull requests ## If you choose to [submit a pull -request](https://github.com/cisagov/postfix-docker/pulls), you will +request](https://github.com/cisagov/skeleton-docker/pulls), you will notice that our continuous integration (CI) system runs a fairly extensive set of linters and syntax checkers. Your pull request may fail these checks, and that's OK. If you want you can stop there and @@ -78,9 +78,9 @@ can create and configure the Python virtual environment with these commands: ```bash -cd postfix-docker -pyenv virtualenv postfix-docker -pyenv local postfix-docker +cd skeleton-docker +pyenv virtualenv skeleton-docker +pyenv local skeleton-docker pip install -r requirements-dev.txt ``` From e5adf50959f5cf2a598bd61d251b30a08592ceb4 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:45:02 -0400 Subject: [PATCH 054/253] Rename project and stow ciphertext --- .travis.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1f956d5..f60659a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,20 +5,20 @@ python: 3.7 services: docker env: global: - - IMAGE_NAME=dhsncats/postfix + - IMAGE_NAME=dhsncats/example - DOCKER_USER=felddy - - secure: >- - R8uSOMb+2aZV/bTK8EU3L5sYqNUiVZpS3kBU0XvVW3rnZgdW1QIHwLU2VOAv9kgpw1e - zXaAi3OPPboi3eMWTXGQGYM3prZePlpDk+cVwCZB5sQn4xw692p/VmNgnxrY9NTts0Y - 0qPSW9YdCEza8RLYbxaar8PGaLDdh5GwYBeBK8wrvd51PQa8J1IKDKRKfsbrKLRjHv8 - d8qSNT2yKa8vX63l80ftwHs07IOO2YKaUjtS4DlON+XHbLqXkfgVqQ8VUntW0qjb1M1 - KBGLckD6/qeCqMAPHDvuJY+fT2n+6Q0tDZbyJhYzYDN9eTTBuetIDoEZSU8vpvDslrG - qH8Pu93x4g63DkMl+lfMm6gNjkB8DLxOlHJvdjBHjiiIsSPdUdSxI/Y7GtkchD/Mkdq - JI5PyQSPnMvLAiNrhysGuIkKnD4hfDyR2RyePFw8zLgKj/Hsm71O1oX7I3aJu08xYL5 - EaXRFkeOMzCPamXNcKjpHWxP7UZqGROBe6e7eOKaJSFzpNZPczgHyi3XxoS8NNPIh0q - 4LTxeqHSUuzyEtg8az3meRiR9z+5LGA7mt2YZ7yyTPwizi2X2LVbx9yCtfnfEG/jrqI - tx6GFgOb7a02jEbDNQ5q0qR56IwFRSlw55EdWMo2gnh2PtJ044QaMzr/du2VM6wydAR - n2KsiL8piwAEc= + # - secure: >- + # R8uSOMb+2aZV/bTK8EU3L5sYqNUiVZpS3kBU0XvVW3rnZgdW1QIHwLU2VOAv9kgpw1e + # zXaAi3OPPboi3eMWTXGQGYM3prZePlpDk+cVwCZB5sQn4xw692p/VmNgnxrY9NTts0Y + # 0qPSW9YdCEza8RLYbxaar8PGaLDdh5GwYBeBK8wrvd51PQa8J1IKDKRKfsbrKLRjHv8 + # d8qSNT2yKa8vX63l80ftwHs07IOO2YKaUjtS4DlON+XHbLqXkfgVqQ8VUntW0qjb1M1 + # KBGLckD6/qeCqMAPHDvuJY+fT2n+6Q0tDZbyJhYzYDN9eTTBuetIDoEZSU8vpvDslrG + # qH8Pu93x4g63DkMl+lfMm6gNjkB8DLxOlHJvdjBHjiiIsSPdUdSxI/Y7GtkchD/Mkdq + # JI5PyQSPnMvLAiNrhysGuIkKnD4hfDyR2RyePFw8zLgKj/Hsm71O1oX7I3aJu08xYL5 + # EaXRFkeOMzCPamXNcKjpHWxP7UZqGROBe6e7eOKaJSFzpNZPczgHyi3XxoS8NNPIh0q + # 4LTxeqHSUuzyEtg8az3meRiR9z+5LGA7mt2YZ7yyTPwizi2X2LVbx9yCtfnfEG/jrqI + # tx6GFgOb7a02jEbDNQ5q0qR56IwFRSlw55EdWMo2gnh2PtJ044QaMzr/du2VM6wydAR + # n2KsiL8piwAEc= cache: pip: true directories: From 57ef61d743fb04952d3e7d6e38d476e663185af9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:45:35 -0400 Subject: [PATCH 055/253] Add example docker composition for testing --- docker-compose.yml | 49 ++++++++++------------------------------------ 1 file changed, 10 insertions(+), 39 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1f0d962..baa392a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,53 +2,24 @@ version: "3.7" secrets: - fullchain_pem: - file: ./src/secrets/fullchain.pem - privkey_pem: - file: ./src/secrets/privkey.pem - users_txt: - file: ./src/secrets/users.txt - + quote_txt: + file: ./src/secrets/quote.txt services: - postfix: + example: build: context: . dockerfile: Dockerfile - image: dhsncats/postfix + image: dhsncats/example init: true - restart: always + restart: on-failure environment: - - PRIMARY_DOMAIN=example.com - - RELAY_IP=172.16.202.1/32 - networks: - front: - ipv4_address: 172.16.202.2 + - ECHO_MESSAGE=Hello World from docker-compose! ports: - - target: "25" - published: "1025" - protocol: tcp - mode: host - - target: "587" - published: "1587" - protocol: tcp - mode: host - - target: "993" - published: "1993" + - target: "8080" + published: "8080" protocol: tcp mode: host secrets: - - source: fullchain_pem - target: fullchain.pem - - source: privkey_pem - target: privkey.pem - - source: users_txt - target: users.txt - -networks: - front: - driver: bridge - ipam: - driver: default - config: - - subnet: 172.16.202.0/24 + - source: quote_txt + target: quote.txt From 55b43a49f199c817ab47a6abab82f6518417acb3 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:52:04 -0400 Subject: [PATCH 056/253] Add sample tests for container using pytest-dockerc --- requirements-test.txt | 1 + tests/conftest.py | 23 +++++++ tests/container_test.py | 145 ++++++---------------------------------- 3 files changed, 45 insertions(+), 124 deletions(-) create mode 100644 tests/conftest.py diff --git a/requirements-test.txt b/requirements-test.txt index 51f1982..fc5504e 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,3 @@ pre-commit pytest +pytest-dockerc diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b060645 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,23 @@ +"""pytest plugin configuration. + +https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins +""" +import pytest + + +def pytest_addoption(parser): + """Add new commandline options to pytest.""" + parser.addoption( + "--runslow", action="store_true", default=False, help="run slow tests" + ) + + +def pytest_collection_modifyitems(config, items): + """Modify collected tests based on custom marks and commandline options.""" + if config.getoption("--runslow"): + # --runslow given in cli: do not skip slow tests + return + skip_slow = pytest.mark.skip(reason="need --runslow option to run") + for item in items: + if "slow" in item.keywords: + item.add_marker(skip_slow) diff --git a/tests/container_test.py b/tests/container_test.py index 0ccd9e9..18de7c7 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,127 +1,24 @@ #!/usr/bin/env pytest -vs -"""Tests for postfix container.""" +"""Tests for example container.""" -from email.message import EmailMessage -from imaplib import IMAP4_SSL -import smtplib - -import pytest - -MESSAGE = """ -This is a test message sent during the unit tests. -""" -DOMAIN = "example.com" -ARCHIVE_USER = "mailarchive" -ARCHIVE_PW = "foobar" -TEST_SEND_USER = "testsender1" -TEST_SEND_PW = "lemmy is god" -IMAP_PORT = 1993 - - -@pytest.mark.parametrize("port", [1025, 1587]) -@pytest.mark.parametrize("to_user", [ARCHIVE_USER, TEST_SEND_USER]) -def test_sending_mail(port, to_user): - """Send an email message to the server.""" - msg = EmailMessage() - msg.set_content(MESSAGE) - msg["Subject"] = f"Test Message on port {port}" - msg["From"] = f"test@{DOMAIN}" - msg["To"] = f"{to_user}@{DOMAIN}" - with smtplib.SMTP("localhost", port=port) as s: - s.send_message(msg) - - -@pytest.mark.parametrize( - "username,password", - [ - (ARCHIVE_USER, ARCHIVE_PW), - (TEST_SEND_USER, TEST_SEND_PW), - pytest.param(ARCHIVE_USER, TEST_SEND_PW, marks=pytest.mark.xfail), - pytest.param("your_mom", "so_fat", marks=pytest.mark.xfail), - ], -) -def test_imap_login(username, password): - """Test logging in to the IMAP server.""" - with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(username, password) - - -@pytest.mark.parametrize( - "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] -) -def test_imap_messages_exist(username, password): - """Test test existence of our test messages.""" - with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(username, password) - typ, data = m.select() - assert typ == "OK", f"Select did not return OK status for {username}" - message_count = int(data[0]) - print(f"{username} inbox message count: {message_count}") - assert message_count > 0, f"Expected message in the {username} inbox" - - -@pytest.mark.parametrize( - "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] -) -def test_imap_reading(username, password): - """Test receiving message from the IMAP server.""" - with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(username, password) - typ, data = m.select() - assert typ == "OK", "Select did not return OK status" - message_count = int(data[0]) - print(f"inbox message count: {message_count}") - typ, data = m.search(None, "ALL") - assert typ == "OK", "Search did not return OK status" - message_numbers = data[0].split() - for num in message_numbers: - typ, data = m.fetch(num, "(RFC822)") - assert typ == "OK", f"Fetch of message {num} did not return OK status" - print("-" * 40) - print(f"Message: {num}") - print(data[0][1].decode("utf-8")) - # mark messag as deleted - typ, data = m.store(num, "+FLAGS", "\\Deleted") - assert ( - typ == "OK" - ), f"Storing '\\deleted' flag on message {num} did not return OK status" - # expunge all deleted messages - typ, data = m.expunge() - assert typ == "OK", "Expunge did not return OK status" - - -@pytest.mark.parametrize( - "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] -) -def test_imap_delete_all(username, password): - """Test deleting messages from the IMAP server.""" - with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(username, password) - typ, data = m.select() - assert typ == "OK", "Select did not return OK status" - typ, data = m.search(None, "ALL") - assert typ == "OK", "Search did not return OK status" - message_numbers = data[0].split() - for num in message_numbers: - # mark messag as deleted - typ, data = m.store(num, "+FLAGS", "\\Deleted") - assert ( - typ == "OK" - ), f"Storing '\\deleted' flag on message {num} did not return OK status" - # expunge all deleted messages - typ, data = m.expunge() - assert typ == "OK", "Expunge did not return OK status" - - -@pytest.mark.parametrize( - "username,password", [(ARCHIVE_USER, ARCHIVE_PW), (TEST_SEND_USER, TEST_SEND_PW)] +SECRET_PW = ( + "There are no secrets better kept than the secrets everybody guesses." # nosec ) -def test_imap_messages_cleared(username, password): - """Test that all messages were expunged.""" - with IMAP4_SSL("localhost", IMAP_PORT) as m: - m.login(username, password) - typ, data = m.select() - assert typ == "OK", "Select did not return OK status" - message_count = int(data[0]) - print(f"inbox message count: {message_count}") - assert message_count == 0, "Expected the inbox to be empty" +ENV_VAR = "ECHO_MESSAGE" +ENV_VAR_VAL = "Hello World from docker-compose!" + + +def test_container(dockerc): + """Verify the test composition and container.""" + assert len(dockerc.containers()) == 1, "Wrong number of containers were running." + container = dockerc.containers()[0] + assert container.is_running is True, "Expected container to be running" + assert container.wait() == 0, "Container did not exit cleanly" + assert ( + container.labels["com.docker.compose.service"] == "example" + ), "Service name was different than expected" + assert ( + container.environment[ENV_VAR] == ENV_VAR_VAL + ), f"{ENV_VAR} value was different than expected" + log_output = container.logs().decode("utf-8") + assert SECRET_PW in log_output, "Secret not found in log output." From dee0fcd9b1a35ab77da14b9dda30eccfa09bdeee Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:52:29 -0400 Subject: [PATCH 057/253] Add a secret quote --- src/secrets/quote.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/secrets/quote.txt diff --git a/src/secrets/quote.txt b/src/secrets/quote.txt new file mode 100644 index 0000000..93ee1a8 --- /dev/null +++ b/src/secrets/quote.txt @@ -0,0 +1 @@ +There are no secrets better kept than the secrets everybody guesses. From 2a6d14888784b73860a9806cf3a4b8fc91c24046 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:53:12 -0400 Subject: [PATCH 058/253] Add dockerfile to build example container --- Dockerfile | 54 ++++++++++++++++++++++-------------------------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6a82707..4f844ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,28 @@ -FROM debian:buster-slim -MAINTAINER Mark Feldhousen +FROM python:3.7-alpine +LABEL maintainer="mark.feldhousen@trio.dhs.gov" -RUN apt-get update && \ -DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ -ca-certificates \ -diceware \ -dovecot-imapd \ -dovecot-lmtpd \ -gettext-base \ -mailutils \ -opendkim \ -opendkim-tools \ -opendmarc \ -postfix \ -procmail \ -sasl2-bin \ -&& apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -RUN adduser mailarchive --quiet --disabled-password \ ---shell /usr/sbin/nologin --gecos "Mail Archive" +ARG CISA_UID=421 +ENV CISA_HOME="/home/cisa" +ENV ECHO_MESSAGE="Hello World from Dockerfile" -USER root -WORKDIR /root +RUN addgroup --system --gid ${CISA_UID} cisa \ + && adduser --system --uid ${CISA_UID} --ingroup cisa cisa -# make backups of configurations. These are modified at startup. -RUN mv /etc/postfix/master.cf /etc/postfix/master.cf.orig -RUN mv /etc/default/opendkim /etc/default/opendkim.orig -RUN mv /etc/default/opendmarc /etc/default/opendmarc.orig +RUN apk --update --no-cache add \ +ca-certificates \ +git \ +openssl \ +py-pip -COPY ./src/templates ./templates/ -COPY ./src/docker-entrypoint.sh . +WORKDIR ${CISA_HOME} +# TODO remove branch +RUN git clone --branch improvement/data https://github.com/cisagov/skeleton-python-library.git . && \ +pip install --requirement requirements.txt && \ +ln -snf /run/secrets/quote.txt src/example/data/secret.txt -VOLUME ["/var/log", "/var/spool/postfix"] -EXPOSE 25/TCP 587/TCP 993/TCP +USER cisa -ENTRYPOINT ["./docker-entrypoint.sh"] -CMD ["postfix", "-v", "start-fg"] +EXPOSE 8080/TCP +VOLUME ["/var/log"] +ENTRYPOINT ["example"] +CMD ["--log-level", "DEBUG"] From c231cc679252062043c937386e4e3696ea110be3 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:53:43 -0400 Subject: [PATCH 059/253] Update README --- README.md | 69 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 20012dd..4e1eaaa 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,60 @@ -# postfix-docker 📮🐳 # +# skeleton-docker 💀🐳 # -[![Build Status](https://travis-ci.com/cisagov/postfix-docker.svg?branch=develop)](https://travis-ci.com/cisagov/postfix-docker) +[![Build Status](https://travis-ci.com/cisagov/skeleton-docker.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-docker) -Creates a Docker container with an installation of the -[postfix](http://postfix.org) MTA. Additionally it has an IMAP -server ([dovecot](https://dovecot.org)) for accessing the archvies -of sent email. All email is BCC'd to the `mailarchive` account. +## Docker Image ## + +![MicroBadger Layers](https://img.shields.io/microbadger/layers/dhsncats/example.svg) +![MicroBadger Size](https://img.shields.io/microbadger/image-size/dhsncats/example.svg) + +This is a docker skeleton project that can be used to quickly get a +new [cisagov](https://github.com/cisagov) GitHub docker project started. +This skeleton project contains [licensing information](LICENSE.md), as +well as [pre-commit hooks](https://pre-commit.com) and a [Travis +CI](https://travis-ci.com) configuration appropriate for docker +containers and the major languages that we use. ## Usage ## -A sample [docker composition](docker-compose.yml) is included in this repository. -To build and start the container use the command: `docker-compose up` +### Install ### -### Ports ### +Pull `dhsncats/example` from the Docker repository: -This container exposes the following ports: + docker pull dhsncats/example -- 25: `smtp` -- 587: `submission` -- 993: `imaps` +Or build `dhsncats/example` from source: -The sample [docker composition](docker-compose.yml) publishes the -exposed ports at 1025, 1587, and 1993. + git clone https://github.com/cisagov/skeleton-docker.git + cd skeleton-docker + docker-compose build -### Environment Variables ### +### Run ### + +## Ports ## + +This container exposes the following ports: -Two environment variables are used to generate the configurations at runtime: +| Port | Protocol | Service | +|-------|----------|----------| +| 8080 | TCP | http | -- `PRIMARY_DOMAIN`: the domain of the mail server -- `RELAY_IP`: (optional) an IP address that is allowed to relay mail without authentication +## Environment Variables ## -### Secrets ### +| Variable | Default Value | Purpose | +|---------------|-------------------------------|--------------| +| ECHO_MESSAGE | `Hello World from Dockerfile` | Text to echo | -- `fullchain.pem`: public key -- `privkey.pem`: private key -- `users.txt`: account credentials to create at startup +## Secrets ## -### Volumes ### +| Filename | Purpose | +|---------------|----------------------| +| quote.txt | Secret text to echo | -Two optional volumes can be attached to this container to persist the -mail spool directory, as well as the logging directory. (Note that -the mail logs are available using the docker log command.) +## Volumes ## -- `/var/spool/postfix`: mail queues -- `/var/log`: system logs +| Mount point | Purpose | +|-------------|----------------| +| /var/log | logging output | ## Contributing ## From 4c3aeda22523270b8ee53ed37b76fa4db5ce9e86 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:55:11 -0400 Subject: [PATCH 060/253] Remove composition up and down. This is now handled by pytest itself --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f60659a..b5a5e27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,13 +26,9 @@ cache: install: - pip install --upgrade --requirement requirements-test.txt - docker build --tag "$IMAGE_NAME" . -before_script: - - docker-compose up --detach script: - pre-commit run --all-files - pytest --verbose -after_script: - - docker-compose down before_deploy: - version=$(./bump_version.sh show) - IFS='.' read -r -a version_array <<< "$version" From 340f6f7809db2d0294b14eb07ed8e48fbdd78d65 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 14:58:47 -0400 Subject: [PATCH 061/253] Update ciphertext for DOCKER_PW --- .travis.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index b5a5e27..861470d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,18 +7,18 @@ env: global: - IMAGE_NAME=dhsncats/example - DOCKER_USER=felddy - # - secure: >- - # R8uSOMb+2aZV/bTK8EU3L5sYqNUiVZpS3kBU0XvVW3rnZgdW1QIHwLU2VOAv9kgpw1e - # zXaAi3OPPboi3eMWTXGQGYM3prZePlpDk+cVwCZB5sQn4xw692p/VmNgnxrY9NTts0Y - # 0qPSW9YdCEza8RLYbxaar8PGaLDdh5GwYBeBK8wrvd51PQa8J1IKDKRKfsbrKLRjHv8 - # d8qSNT2yKa8vX63l80ftwHs07IOO2YKaUjtS4DlON+XHbLqXkfgVqQ8VUntW0qjb1M1 - # KBGLckD6/qeCqMAPHDvuJY+fT2n+6Q0tDZbyJhYzYDN9eTTBuetIDoEZSU8vpvDslrG - # qH8Pu93x4g63DkMl+lfMm6gNjkB8DLxOlHJvdjBHjiiIsSPdUdSxI/Y7GtkchD/Mkdq - # JI5PyQSPnMvLAiNrhysGuIkKnD4hfDyR2RyePFw8zLgKj/Hsm71O1oX7I3aJu08xYL5 - # EaXRFkeOMzCPamXNcKjpHWxP7UZqGROBe6e7eOKaJSFzpNZPczgHyi3XxoS8NNPIh0q - # 4LTxeqHSUuzyEtg8az3meRiR9z+5LGA7mt2YZ7yyTPwizi2X2LVbx9yCtfnfEG/jrqI - # tx6GFgOb7a02jEbDNQ5q0qR56IwFRSlw55EdWMo2gnh2PtJ044QaMzr/du2VM6wydAR - # n2KsiL8piwAEc= + - secure: >- + GWxja6rCcevNRay3uOGEt1wpzGOAoKgfquyror60VtkG07GCJ/rCBa/jNsTRRwwDxYW + MqY6xQlAU+GzNeuBxKAlmyrLRRYZDmcBpPgQQPbiC0b5ftGCzTuhdYAj4Hg+LyHKbNP + H0zsXfQRy2NdWPJ8rr3RFL89a/If0hJVL5OEEHfmzpfvHoDvBJQwhmL2sPZizgCsgh6 + eRStRvUWj3taVb5VZ1uIlet8P3g7azkD7tTNx+wAxhjlLpaQfBcy/CC27ogw+QOvgdF + kefskYz54LLzTyXYhRhACjuGaukpBvQBFzBi3MOFYoLj4kb+9b3mqHlnQZLOSo2zlWV + 30mTC7eed5KE64MuQ6I1E72B0rnAoVpnwAjXVIvzfhKBuzbhS54tWjKbqK6UAGxbg6t + wJDiNaB+nYk6zozJ93TQT02XDUOdDS45hO2Kc/CUwJbrZc7V4aYN6Dj30tWy8d2TI8s + GMPuJm8tyxpT/h2eUUPhVzhMCk+4efIBa6cb99wZgkphLmWQefLLpZy8pMwOH7c3ywU + +6tVPBZxSEm4tFRYyPwgl9qvzE8E0WM3FPNgwVO9aypx61HcO8NVD7alUW2j+s3taHX + aVpaCtsf+bpsXPro3/ofst+FHiFPZGBL2iiSIYtFYI4hkDFq9iGGGexo9zgYS6CMexu + YoZtThDw6kaLI= cache: pip: true directories: From 5cd2569407fed07f1b90c7a40820fb4f5f94c518 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 15:08:58 -0400 Subject: [PATCH 062/253] Remove branch checkout --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4f844ce..033fe0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,8 +15,8 @@ openssl \ py-pip WORKDIR ${CISA_HOME} -# TODO remove branch -RUN git clone --branch improvement/data https://github.com/cisagov/skeleton-python-library.git . && \ + +RUN git clone https://github.com/cisagov/skeleton-python-library.git . && \ pip install --requirement requirements.txt && \ ln -snf /run/secrets/quote.txt src/example/data/secret.txt From b58fa62c004d4ffe42188f258653211ee85d6405 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 6 Jun 2019 16:12:58 -0400 Subject: [PATCH 063/253] Add changes from running pre-commit autoupdate --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6a0939..e9029c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,7 +45,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.17.1 + rev: v1.18.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -64,7 +64,7 @@ repos: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.11.0 + rev: v1.12.0 hooks: - id: terraform_fmt - id: terraform_validate_no_variables From 740457694ab4f14fbe89583f14de86aab064765b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 6 Jun 2019 16:33:07 -0400 Subject: [PATCH 064/253] Remove the terraform-docs hook terraform-docs is currently broken with respect to terraform 0.12. --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9029c0..dd98ddc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,7 +68,6 @@ repos: hooks: - id: terraform_fmt - id: terraform_validate_no_variables - - id: terraform_docs - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.0 hooks: From 0d10121e65a1083c56eb6720338f8ce490a14e47 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 6 Jun 2019 18:52:59 -0400 Subject: [PATCH 065/253] Add version label --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 033fe0b..6cc894e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ FROM python:3.7-alpine LABEL maintainer="mark.feldhousen@trio.dhs.gov" +LABEL version="0.0.1" ARG CISA_UID=421 ENV CISA_HOME="/home/cisa" From bd4206bbcd94f00aaa3ae8a4b1daf8db002120f1 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 7 Jun 2019 11:19:41 -0400 Subject: [PATCH 066/253] Configure prettier to ignore JSON files JSON files are already being linted by the pretty-format-json hook. --- .prettierignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.prettierignore b/.prettierignore index f1e840c..738d402 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,5 @@ +# Already being linted by pretty-format-json +*.json # Already being linted by mdl *.md # Already being linted by yamllint From 365fff926c7c11d266d4cc64148f2d2d9b283245 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 11:28:22 -0400 Subject: [PATCH 067/253] Add robust container tests --- tests/conftest.py | 21 ++++++++++++++++++ tests/container_test.py | 49 +++++++++++++++++++++++++++++------------ 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b060645..d62dff6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,27 @@ import pytest +MAIN_SERVICE_NAME = "example" +VERSION_SERVICE_NAME = f"{MAIN_SERVICE_NAME}-version" + + +@pytest.fixture(scope="session") +def main_container(dockerc): + """Return the main container from the docker composition.""" + # find the container by name even if it is stopped already + return dockerc.containers(service_names=[MAIN_SERVICE_NAME], stopped=True)[0] + + +@pytest.fixture(scope="session") +def version_container(dockerc): + """Return the version container from the docker composition. + + The version container should just output the version of its underlying contents. + """ + # find the container by name even if it is stopped already + return dockerc.containers(service_names=[VERSION_SERVICE_NAME], stopped=True)[0] + + def pytest_addoption(parser): """Add new commandline options to pytest.""" parser.addoption( diff --git a/tests/container_test.py b/tests/container_test.py index 18de7c7..494ad45 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,24 +1,45 @@ #!/usr/bin/env pytest -vs """Tests for example container.""" -SECRET_PW = ( - "There are no secrets better kept than the secrets everybody guesses." # nosec -) ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker-compose!" +SECRET_QUOTE = ( + "There are no secrets better kept than the secrets everybody guesses." # nosec +) +VERSION_FILE = "src/version.txt" -def test_container(dockerc): +def test_container_count(dockerc): """Verify the test composition and container.""" - assert len(dockerc.containers()) == 1, "Wrong number of containers were running." - container = dockerc.containers()[0] - assert container.is_running is True, "Expected container to be running" - assert container.wait() == 0, "Container did not exit cleanly" + assert len(dockerc.containers()) == 2, "Wrong number of containers were running." + + +def test_wait_for_exits(main_container, version_container): + """Wait for containers to exit.""" + assert main_container.wait() == 0, "Container service (main) did not exit cleanly" + assert ( + version_container.wait() == 0 + ), "Container service (version) did not exit cleanly" + + +def test_output(main_container): + """Verify the container had the correct output.""" + main_container.wait() # make sure container exited if running test isolated + log_output = main_container.logs().decode("utf-8") + assert SECRET_QUOTE in log_output, "Secret not found in log output." + + +def test_version_matches(version_container): + """Verify the container outputs the correct version.""" + version_container.wait() # make sure container exited if running test isolated + log_output = version_container.logs().decode("utf-8").strip() + pkg_vars = {} + with open(VERSION_FILE) as f: + exec(f.read(), pkg_vars) # nosec + project_version = pkg_vars["__version__"] assert ( - container.labels["com.docker.compose.service"] == "example" - ), "Service name was different than expected" + log_output == project_version + ), f"Container version output does not match project version file {VERSION_FILE}" assert ( - container.environment[ENV_VAR] == ENV_VAR_VAL - ), f"{ENV_VAR} value was different than expected" - log_output = container.logs().decode("utf-8") - assert SECRET_PW in log_output, "Secret not found in log output." + version_container.labels["version"] == project_version + ), "Dockerfile version label does not match project version" From adcf39e6e86c105ba0c02072a8accdee704734a8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 11:28:45 -0400 Subject: [PATCH 068/253] Add second instance of container for version test --- docker-compose.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index baa392a..8579d0a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,21 @@ --- version: "3.7" +# This docker-compose file is used to build and test the container + secrets: quote_txt: file: ./src/secrets/quote.txt services: example: + # Run the container normally build: context: . dockerfile: Dockerfile image: dhsncats/example init: true - restart: on-failure + restart: "no" environment: - ECHO_MESSAGE=Hello World from docker-compose! ports: @@ -23,3 +26,10 @@ services: secrets: - source: quote_txt target: quote.txt + + example-version: + # Run the container to collect version information + image: dhsncats/example + init: true + restart: "no" + command: --version From b63c80ffc29c85a24b0a08f2b8ae9aeb610d0011 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 11:29:22 -0400 Subject: [PATCH 069/253] Add useful labels --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6cc894e..88a6a82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ FROM python:3.7-alpine +ARG VERSION="0.0.1" LABEL maintainer="mark.feldhousen@trio.dhs.gov" -LABEL version="0.0.1" +LABEL vendor="Cyber and Infrastructure Security Agency" +LABEL version=${VERSION} ARG CISA_UID=421 ENV CISA_HOME="/home/cisa" From 5b0d395d1f47be2c3264337430908707c4d2cb2d Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 11:29:58 -0400 Subject: [PATCH 070/253] Use release of code instead of repo --- Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 88a6a82..b4dd7f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,15 +13,16 @@ RUN addgroup --system --gid ${CISA_UID} cisa \ RUN apk --update --no-cache add \ ca-certificates \ -git \ openssl \ py-pip WORKDIR ${CISA_HOME} -RUN git clone https://github.com/cisagov/skeleton-python-library.git . && \ -pip install --requirement requirements.txt && \ -ln -snf /run/secrets/quote.txt src/example/data/secret.txt +RUN wget -O sourcecode.tgz https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz && \ + tar xzf sourcecode.tgz --strip-components=1 && \ + pip install --requirement requirements.txt && \ + ln -snf /run/secrets/quote.txt src/example/data/secret.txt && \ + rm sourcecode.tgz USER cisa From 8380ecd53b6b4a6f9103c2eb776241a5f1ff8a0b Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:25:16 -0400 Subject: [PATCH 071/253] Expose ARGs for external configuration --- Dockerfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b4dd7f9..ba75023 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,15 @@ +ARG GIT_COMMIT=unspecified +ARG GIT_REMOTE=unspecified +ARG VERSION=unspecified + FROM python:3.7-alpine -ARG VERSION="0.0.1" + +ARG GIT_COMMIT +ARG GIT_REMOTE +ARG VERSION + +LABEL git_commit=${GIT_COMMIT} +LABEL git_remote=${GIT_REMOTE} LABEL maintainer="mark.feldhousen@trio.dhs.gov" LABEL vendor="Cyber and Infrastructure Security Agency" LABEL version=${VERSION} From 17751059c7e461777be8327edbc013ef940988ed Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:26:47 -0400 Subject: [PATCH 072/253] Update to show version specification for manual build --- README.md | 2 +- docker-compose.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e1eaaa..ac29e5c 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Or build `dhsncats/example` from source: git clone https://github.com/cisagov/skeleton-docker.git cd skeleton-docker - docker-compose build + docker-compose build --build-arg VERSION=0.0.1 ### Run ### diff --git a/docker-compose.yml b/docker-compose.yml index 8579d0a..f9a54ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ services: example: # Run the container normally build: + # VERSION must be specified on the command line: + # e.g., --build-arg VERSION=0.0.1 context: . dockerfile: Dockerfile image: dhsncats/example From f40541cd8d6ae567df08e7ec694860eb824ea2c8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:27:19 -0400 Subject: [PATCH 073/253] Add build arguments for image creation --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 861470d..a06bc93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,12 +25,17 @@ cache: - "$HOME/.cache/pre-commit" install: - pip install --upgrade --requirement requirements-test.txt - - docker build --tag "$IMAGE_NAME" . + - version=$(./bump_version.sh show) + - docker build + --tag "$IMAGE_NAME" + --build-arg GIT_COMMIT=$(git log -1 --format=%H) + --build-arg GIT_REMOTE=$(git remote get-url origin) + --build-arg VERSION=${version} + . script: - pre-commit run --all-files - pytest --verbose before_deploy: - - version=$(./bump_version.sh show) - IFS='.' read -r -a version_array <<< "$version" - docker login --username "$DOCKER_USER" --password "$DOCKER_PW" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" From befbec0e6163845d1393b06bbabd6493c04cefb4 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:36:27 -0400 Subject: [PATCH 074/253] Fix for travis being a bit pokey at startup --- tests/container_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/container_test.py b/tests/container_test.py index 494ad45..b6835aa 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -11,7 +11,10 @@ def test_container_count(dockerc): """Verify the test composition and container.""" - assert len(dockerc.containers()) == 2, "Wrong number of containers were running." + # stopped parameter allows non-running containers in results + assert ( + len(dockerc.containers(stopped=True)) == 2 + ), "Wrong number of containers were started." def test_wait_for_exits(main_container, version_container): From 7bd93f1ef735a1ba1025f6c7d5a4198172234f2f Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:50:01 -0400 Subject: [PATCH 075/253] Add version check for TRAVIS_TAG when available --- tests/container_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/container_test.py b/tests/container_test.py index b6835aa..fd7b085 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,6 +1,8 @@ #!/usr/bin/env pytest -vs """Tests for example container.""" +import os + ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker-compose!" SECRET_QUOTE = ( @@ -46,3 +48,8 @@ def test_version_matches(version_container): assert ( version_container.labels["version"] == project_version ), "Dockerfile version label does not match project version" + travis_tag = os.getenv("TRAVIS_TAG") + if travis_tag is not None: + assert ( + travis_tag == project_version + ), "TRAVIS_TAG does not match the project version" From c88c5dc535645f0128254248f3e042d2387aa718 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 12:54:15 -0400 Subject: [PATCH 076/253] Travis you biatch! --- tests/container_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/container_test.py b/tests/container_test.py index fd7b085..45a9588 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -49,7 +49,7 @@ def test_version_matches(version_container): version_container.labels["version"] == project_version ), "Dockerfile version label does not match project version" travis_tag = os.getenv("TRAVIS_TAG") - if travis_tag is not None: + if travis_tag not in [None, ""]: assert ( travis_tag == project_version ), "TRAVIS_TAG does not match the project version" From 285d520ca239f3bc1d12b3ad8fb1c005f7cde38c Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 13:05:08 -0400 Subject: [PATCH 077/253] Travis you biatch!! --- tests/container_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/container_test.py b/tests/container_test.py index 45a9588..4c80ca8 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -49,7 +49,7 @@ def test_version_matches(version_container): version_container.labels["version"] == project_version ), "Dockerfile version label does not match project version" travis_tag = os.getenv("TRAVIS_TAG") - if travis_tag not in [None, ""]: + if travis_tag: assert ( travis_tag == project_version ), "TRAVIS_TAG does not match the project version" From 4913c2bece0d780db30aa497a97603827e2476b8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 13:15:50 -0400 Subject: [PATCH 078/253] =?UTF-8?q?You're=20not=20wrong=20Travis,=20you're?= =?UTF-8?q?=20just=20an=20=F0=9F=90=B4=F0=9F=95=B3.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/container_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/container_test.py b/tests/container_test.py index 4c80ca8..216d392 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -51,5 +51,5 @@ def test_version_matches(version_container): travis_tag = os.getenv("TRAVIS_TAG") if travis_tag: assert ( - travis_tag == project_version + travis_tag == project_version or travis_tag == f"v{project_version}" ), "TRAVIS_TAG does not match the project version" From fb35101cd7f6a8f206594eaf5cf98ebc03004c02 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 7 Jun 2019 14:57:40 -0400 Subject: [PATCH 079/253] Add missing run instructions --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ac29e5c..aa74a59 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ Or build `dhsncats/example` from source: ### Run ### + docker-compose run --rm example + ## Ports ## This container exposes the following ports: From c6f2adb6abc41b0bbb6abae88aba10cb6ab2e515 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell Date: Thu, 20 Jun 2019 15:14:12 -0400 Subject: [PATCH 080/253] Update two text blocks to use the console tag instead of the bash tag for formatting. --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ede2f47..117baba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,7 +77,7 @@ Once `pyenv` and `pyenv-virtualenv` are installed on your system, you can create and configure the Python virtual environment with these commands: -```bash +```console cd skeleton-generic pyenv virtualenv skeleton-generic pyenv local skeleton-generic @@ -88,7 +88,7 @@ pip install -r requirements-dev.txt Now setting up pre-commit is as simple as: -```bash +```console pre-commit install ``` From d90ce5509cd7da08b7d5187481561133a8bbc1dd Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Jun 2019 15:24:46 -0400 Subject: [PATCH 081/253] Add lgtm configuration for requirements-test.txt file --- .lgtm.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .lgtm.yml diff --git a/.lgtm.yml b/.lgtm.yml new file mode 100644 index 0000000..8950263 --- /dev/null +++ b/.lgtm.yml @@ -0,0 +1,8 @@ +--- +extraction: + python: + python_setup: + version: 3 + requirements_files: + - requirements-test.txt + setup_py: false From 06a2b906cef6a8f25677b251b14806b6473d12ed Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Jun 2019 15:25:32 -0400 Subject: [PATCH 082/253] Add LGTM shields --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index aa74a59..0bd4127 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # skeleton-docker 💀🐳 # [![Build Status](https://travis-ci.com/cisagov/skeleton-docker.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-docker) +[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/alerts/) +[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/context:python) ## Docker Image ## From a13597d2330f947797fce78bed3ac665c0623ddc Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 24 Jun 2019 17:11:45 -0400 Subject: [PATCH 083/253] Update pre-commit hooks via pre-commit autoupdate --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd98ddc..eeddaed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.16.0 + rev: v0.17.0 hooks: - id: markdownlint # The LICENSE.md must match the license text exactly for @@ -31,7 +31,7 @@ repos: # alone. exclude: LICENSE.md - repo: https://github.com/adrienverge/yamllint - rev: v1.15.0 + rev: v1.16.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -45,11 +45,11 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.18.0 + rev: v1.19.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit - rev: 1.6.0 + rev: 1.6.1 hooks: - id: bandit args: @@ -73,6 +73,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.17.1 + rev: 1.18.2 hooks: - id: prettier From 1843ca0b482624b867828c95c99faa9cb01253b7 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 24 Jun 2019 17:14:02 -0400 Subject: [PATCH 084/253] Add config file for markdownlint This config file turns off the "line too long" rule when within a code block or a table. Often there is no way to avoid going over 80 lines in such cases. --- .mdl_config.json | 7 +++++++ .pre-commit-config.yaml | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 .mdl_config.json diff --git a/.mdl_config.json b/.mdl_config.json new file mode 100644 index 0000000..492955a --- /dev/null +++ b/.mdl_config.json @@ -0,0 +1,7 @@ +{ + "MD013": { + "code_blocks": false, + "tables": false + }, + "default": true +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eeddaed..491b94c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,6 +30,8 @@ repos: # GitHub's autorecognition fu to work, so we should leave it # alone. exclude: LICENSE.md + args: + - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint rev: v1.16.0 hooks: From 3fee0b86874070e9c9cb41a6d82b7904a3352590 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 24 Jun 2019 17:17:54 -0400 Subject: [PATCH 085/253] Rename LICENSE.md to LICENSE, make the other changes that requires The license text is not actually in Markdown, and adding the .md extension actually makes it render a little differently than intended. --- .pre-commit-config.yaml | 4 ---- CONTRIBUTING.md | 4 ++-- LICENSE.md => LICENSE | 0 README.md | 4 ++-- 4 files changed, 4 insertions(+), 8 deletions(-) rename LICENSE.md => LICENSE (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 491b94c..a4b6299 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,10 +26,6 @@ repos: rev: v0.17.0 hooks: - id: markdownlint - # The LICENSE.md must match the license text exactly for - # GitHub's autorecognition fu to work, so we should leave it - # alone. - exclude: LICENSE.md args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 117baba..93addc2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,8 +8,8 @@ of contribution, and don't want a wall of rules to get in the way of that. Before contributing, we encourage you to read our CONTRIBUTING policy -(you are here), our [LICENSE](LICENSE.md), and our -[README](README.md), all of which should be in this repository. +(you are here), our [LICENSE](LICENSE), and our [README](README.md), +all of which should be in this repository. ## Issues ## diff --git a/LICENSE.md b/LICENSE similarity index 100% rename from LICENSE.md rename to LICENSE diff --git a/README.md b/README.md index 2337970..b880f4d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. -This skeleton project contains [licensing information](LICENSE.md), as +This skeleton project contains [licensing information](LICENSE), as well as [pre-commit hooks](https://pre-commit.com) and a [Travis CI](https://travis-ci.com) configuration appropriate for the major languages that we use. @@ -19,7 +19,7 @@ details. ## License ## -This project is in the worldwide [public domain](LICENSE.md). +This project is in the worldwide [public domain](LICENSE). This project is in the public domain within the United States, and copyright and related rights in the work worldwide are waived through From 74ff0acee52e57f6d82458bc6ab4264701d5dd79 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 27 Jun 2019 11:32:50 -0400 Subject: [PATCH 086/253] Add test to check for container readiness. --- tests/container_test.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/container_test.py b/tests/container_test.py index 216d392..c1cf95e 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -2,9 +2,11 @@ """Tests for example container.""" import os +import time ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker-compose!" +READY_MESSAGE = "This is a debug message" SECRET_QUOTE = ( "There are no secrets better kept than the secrets everybody guesses." # nosec ) @@ -19,6 +21,20 @@ def test_container_count(dockerc): ), "Wrong number of containers were started." +def test_wait_for_ready(main_container): + """Wait for container to be ready.""" + TIMEOUT = 10 + for i in range(TIMEOUT): + if READY_MESSAGE in main_container.logs().decode("utf-8"): + break + time.sleep(1) + else: + raise Exception( + f"Container does not seem ready. " + f'Expected "{READY_MESSAGE}" in the log within {TIMEOUT} seconds.' + ) + + def test_wait_for_exits(main_container, version_container): """Wait for containers to exit.""" assert main_container.wait() == 0, "Container service (main) did not exit cleanly" From 2c6377123f3600ee8c8cec2d6c7298413a07a613 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 27 Jun 2019 17:07:16 -0400 Subject: [PATCH 087/253] Add script to push readme to Docker Hub. Snarfed from: https://github.com/moikot/golang-dep/blob/aab3ea8462a19407544f1ce9daa11c3f0924394c/.travis/push.sh#L127 --- push_readme.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 push_readme.sh diff --git a/push_readme.sh b/push_readme.sh new file mode 100755 index 0000000..880b95b --- /dev/null +++ b/push_readme.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# push the README.md file to the docker hub repository + +set -o nounset +set -o errexit +set -o pipefail + +token=$(curl -s -X POST \ + -H "Content-Type: application/json" \ + -d '{"username": "'"$DOCKER_USER"'", "password": "'"$DOCKER_PW"'"}' \ + https://hub.docker.com/v2/users/login/ | jq -r .token) + +code=$(jq -n --arg msg "$( Date: Thu, 27 Jun 2019 17:07:57 -0400 Subject: [PATCH 088/253] Add comment to identify ciphertext blob purpose --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a06bc93..112e965 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ env: global: - IMAGE_NAME=dhsncats/example - DOCKER_USER=felddy + # DOCKER_PW - secure: >- GWxja6rCcevNRay3uOGEt1wpzGOAoKgfquyror60VtkG07GCJ/rCBa/jNsTRRwwDxYW MqY6xQlAU+GzNeuBxKAlmyrLRRYZDmcBpPgQQPbiC0b5ftGCzTuhdYAj4Hg+LyHKbNP From 28ecdae1f3e12c5d1f7f358fa3baea6391ea79dc Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 27 Jun 2019 17:08:39 -0400 Subject: [PATCH 089/253] Add call to push_readme script --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 112e965..cedc956 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,8 @@ deploy: script: docker push "${IMAGE_NAME}:latest" && docker push "${IMAGE_NAME}:${version}" && docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" && - docker push "${IMAGE_NAME}:${version_array[0]}" + docker push "${IMAGE_NAME}:${version_array[0]}" && + ./push_readme.sh on: tags: true python: '3.7' From 60f495c69ed5c524177c81c848d687a42fa0feeb Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 27 Jun 2019 17:10:54 -0400 Subject: [PATCH 090/253] Document script inputs --- push_readme.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/push_readme.sh b/push_readme.sh index 880b95b..0b6d07a 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash -# push the README.md file to the docker hub repository +# Push the README.md file to the docker hub repository + +# Requires the following environment variables to be set: +# DOCKER_PW, DOCKER_USER, IMAGE_NAME set -o nounset set -o errexit From cb8e8ffd5d07f248f6b39d433620f0207df69fb7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 19 Jul 2019 15:45:57 -0400 Subject: [PATCH 091/253] fix issue 5 --- tests/container_test.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/container_test.py b/tests/container_test.py index c1cf95e..50b62d8 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -4,12 +4,15 @@ import os import time +import pytest + ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker-compose!" READY_MESSAGE = "This is a debug message" SECRET_QUOTE = ( "There are no secrets better kept than the secrets everybody guesses." # nosec ) +TRAVIS_TAG = os.getenv("TRAVIS_TAG") VERSION_FILE = "src/version.txt" @@ -50,8 +53,22 @@ def test_output(main_container): assert SECRET_QUOTE in log_output, "Secret not found in log output." -def test_version_matches(version_container): - """Verify the container outputs the correct version.""" +@pytest.mark.skipif( + TRAVIS_TAG in [None, ""], reason="this is not a release (TRAVIS_TAG not set)" +) +def test_release_version(): + """Verify that release tag version agrees with the module version.""" + pkg_vars = {} + with open(VERSION_FILE) as f: + exec(f.read(), pkg_vars) # nosec + project_version = pkg_vars["__version__"] + assert ( + TRAVIS_TAG == f"v{project_version}" + ), "TRAVIS_TAG does not match the project version" + + +def test_log_version(version_container): + """Verify the container outputs the correct version to the logs.""" version_container.wait() # make sure container exited if running test isolated log_output = version_container.logs().decode("utf-8").strip() pkg_vars = {} @@ -60,12 +77,15 @@ def test_version_matches(version_container): project_version = pkg_vars["__version__"] assert ( log_output == project_version - ), f"Container version output does not match project version file {VERSION_FILE}" + ), f"Container version output to log does not match project version file {VERSION_FILE}" + + +def test_container_version_label_matches(version_container): + """Verify the container version label is the correct version.""" + pkg_vars = {} + with open(VERSION_FILE) as f: + exec(f.read(), pkg_vars) # nosec + project_version = pkg_vars["__version__"] assert ( version_container.labels["version"] == project_version ), "Dockerfile version label does not match project version" - travis_tag = os.getenv("TRAVIS_TAG") - if travis_tag: - assert ( - travis_tag == project_version or travis_tag == f"v{project_version}" - ), "TRAVIS_TAG does not match the project version" From 2932d35483cb8b7be71da69b7006e58298846792 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 11 Sep 2019 15:58:44 -0400 Subject: [PATCH 092/253] Add updates from running pre-commit autoupdate --- .pre-commit-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a4b6299..9580afa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.2.3 + rev: v2.3.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,13 +23,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.17.0 + rev: v0.18.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.16.0 + rev: v1.17.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -37,17 +37,17 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.7 + rev: 3.7.8 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.19.0 + rev: v1.23.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit - rev: 1.6.1 + rev: 1.6.2 hooks: - id: bandit args: @@ -57,7 +57,7 @@ repos: hooks: - id: black - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.0a0 + rev: v4.1.1a0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From 78a230d46c532176c0aa1af70b33f3bca4664243 Mon Sep 17 00:00:00 2001 From: Dave Redmin Date: Fri, 18 Oct 2019 10:10:19 -0400 Subject: [PATCH 093/253] Add updates from running pre-commit autoupdate --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9580afa..0057d09 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.17.0 + rev: v1.18.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -43,7 +43,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.23.0 + rev: v1.25.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit From 05a136bb117b9fb966e0ce3bd0d9784a32f08c51 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 10:39:22 -0400 Subject: [PATCH 094/253] Replace Travis-CI with GitHub actions. --- .github/workflows/build.yml | 26 ++++++++++++++++++++++++++ .travis.yml | 17 ----------------- README.md | 8 ++++---- 3 files changed, 30 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..b3e5b2f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,26 @@ +--- +name: build + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade -r requirements-test.txt + + - name: Run pre-commit on all files + run: pre-commit run --all-files + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2c70e21..0000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -dist: xenial -language: python -python: 3.7 -# pre-commit hooks can use Docker, so we should go ahead and enable it -services: docker - -# Cache pip packages and pre-commit plugins to speed up builds -cache: - pip: true - directories: - - $HOME/.cache/pre-commit - -install: - - pip install --upgrade -r requirements-test.txt -script: - - pre-commit run --all-files diff --git a/README.md b/README.md index b880f4d..36c7336 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # skeleton-generic # -[![Build Status](https://travis-ci.com/cisagov/skeleton-generic.svg?branch=develop)](https://travis-ci.com/cisagov/skeleton-generic) +[![GitHub Build Status](https://github.com/cisagov/skeleton-generic/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-python-library/actions) This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. This skeleton project contains [licensing information](LICENSE), as -well as [pre-commit hooks](https://pre-commit.com) and a [Travis -CI](https://travis-ci.com) configuration appropriate for the major -languages that we use. +well as [pre-commit hooks](https://pre-commit.com) and +[GitHub Actions](https://github.com/features/actions) configurations +appropriate for the major languages that we use. In many cases you will instead want to use one of the more specific skeleton projects derived from this one. From 02fb008d62ca07a78a0103a7af1d81b4712613c1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 10:40:58 -0400 Subject: [PATCH 095/253] Fix action shield link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 36c7336..606df0a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # skeleton-generic # -[![GitHub Build Status](https://github.com/cisagov/skeleton-generic/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-python-library/actions) +[![GitHub Build Status](https://github.com/cisagov/skeleton-generic/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-generic/actions) This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub project started. From b31ad8591f030b2300b82775dfce2fb3c85fc8db Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 10:46:32 -0400 Subject: [PATCH 096/253] Disable ansible-lint until they fix their setup.py See: https://github.com/ansible/ansible-lint/issues/590 --- .pre-commit-config.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0057d09..fca1e26 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,11 +56,12 @@ repos: rev: 19.3b0 hooks: - id: black - - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a0 - hooks: - - id: ansible-lint - # files: molecule/default/playbook.yml + # Disabled until: https://github.com/ansible/ansible-lint/issues/590 + # - repo: https://github.com/ansible/ansible-lint.git + # rev: v4.1.1a0 + # hooks: + # - id: ansible-lint + # # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: From b11e39abd7e62b93f9a14158919bd524317ff34e Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 18 Oct 2019 11:00:39 -0400 Subject: [PATCH 097/253] Remove extra line. --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b3e5b2f..013abd4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,4 +23,3 @@ jobs: - name: Run pre-commit on all files run: pre-commit run --all-files - From 3c78dd98a6ee54b1a102f4e4bddb6c0a31cf4c1d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 2 Nov 2019 19:48:42 -0400 Subject: [PATCH 098/253] Re-enabled ansible-lint and ran pre-commit autoupgrade. --- .pre-commit-config.yaml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fca1e26..6d7bcbc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ --- repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 + rev: v2.4.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -23,7 +23,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.18.0 + rev: v0.19.0 hooks: - id: markdownlint args: @@ -37,13 +37,13 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.8 + rev: 3.7.9 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.25.0 + rev: v1.25.1 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -53,15 +53,14 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/python/black - rev: 19.3b0 + rev: 19.10b0 hooks: - id: black - # Disabled until: https://github.com/ansible/ansible-lint/issues/590 - # - repo: https://github.com/ansible/ansible-lint.git - # rev: v4.1.1a0 - # hooks: - # - id: ansible-lint - # # files: molecule/default/playbook.yml + - repo: https://github.com/ansible/ansible-lint.git + rev: v4.1.1a3 + hooks: + - id: ansible-lint + # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: From 5d5567d303c8ef4c1074ba86bfc7a73dad63b4c8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 5 Nov 2019 14:27:20 -0500 Subject: [PATCH 099/253] Add GitHub action caching of pre-commit hooks and pip packages. --- .github/workflows/build.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 013abd4..fe47599 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,9 +5,7 @@ on: [push] jobs: build: - runs-on: ubuntu-latest - steps: - uses: actions/checkout@v1 @@ -16,6 +14,25 @@ jobs: with: python-version: 3.7 + - name: Cache pre-commit hooks + uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: "${{ runner.os }}-pre-commit-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" + restore-keys: | + ${{ runner.os }}-pre-commit- + + - name: Cache pip test requirements + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" + restore-keys: | + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- + - name: Install dependencies run: | python -m pip install --upgrade pip From 99c611557e432fe6615f7495b6dbdf5c78ac9ae6 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 5 Nov 2019 14:51:49 -0500 Subject: [PATCH 100/253] Remove search for previous pre-commit caches. These seem to be very large caches. Restoring an old one and updating it results in a cache larger than the maximum allowed cache size: 200MB. "Cache size of 254757924 bytes is over the 200MB limit, not saving cache." So if the config changes it is best to just take the cache-miss and start from scratch. --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe47599..d12843d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,8 +20,6 @@ jobs: path: ~/.cache/pre-commit key: "${{ runner.os }}-pre-commit-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - restore-keys: | - ${{ runner.os }}-pre-commit- - name: Cache pip test requirements uses: actions/cache@v1 From 398e7f5ded6b896cf970b69d2024d858260f3e34 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 12 Nov 2019 11:09:52 -0500 Subject: [PATCH 101/253] Add new workflow: lint, test, and build artifact. --- .github/workflows/build.yml | 71 ++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d12843d..df36bee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,38 +3,85 @@ name: build on: [push] +env: + IMAGE_NAME: dhsncats/example + PIP_CACHE_DIR: ~/.cache/pip + PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + + jobs: - build: + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - uses: actions/setup-python@v1 with: python-version: 3.7 - + - name: Cache pip test requirements + uses: actions/cache@v1 + with: + path: ${{ env.PIP_CACHE_DIR }} + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" + restore-keys: | + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- - name: Cache pre-commit hooks uses: actions/cache@v1 with: - path: ~/.cache/pre-commit + path: ${{ env.PRE_COMMIT_CACHE_DIR }} key: "${{ runner.os }}-pre-commit-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade -r requirements-test.txt + - name: Run linters on all files + run: pre-commit run --all-files + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 - name: Cache pip test requirements uses: actions/cache@v1 with: - path: ~/.cache/pip + path: ${{ env.PIP_CACHE_DIR }} key: "${{ runner.os }}-pip-test-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- - - name: Install dependencies run: | python -m pip install --upgrade pip pip install --upgrade -r requirements-test.txt - - - name: Run pre-commit on all files - run: pre-commit run --all-files + - name: Run tests + run: pytest + build: + runs-on: ubuntu-latest + needs: [lint, test] + steps: + - uses: actions/checkout@v1 + - name: Build docker image + run: | + version=$(./bump_version.sh show) + docker build \ + --tag "$IMAGE_NAME" \ + --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ + --build-arg GIT_REMOTE=$(git remote get-url origin) \ + --build-arg VERSION=${version} \ + . + - name: Save docker image + run: | + mkdir dist + version=$(./bump_version.sh show) + docker save $IMAGE_NAME:latest | \ + gzip > dist/"${IMAGE_NAME}"_"${version}".tar.gz + - name: Upload artifacts + uses: actions/upload-artifact@v1 + with: + name: dist + path: dist From 6bdab7d11cdb371d0612392d4f8ec34a7a01ef87 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 12 Nov 2019 11:50:22 -0500 Subject: [PATCH 102/253] Add release workflow based on Travis. --- .github/workflows/release.yml | 53 +++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..8fcc8e1 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,53 @@ +--- +name: release + +on: + release: + types: [prereleased, released] + +env: + IMAGE_NAME: dhsncats/example + DOCKER_PW: ${{ secrets.DOCKER_PW }} + DOCKER_USER: ${{ secrets.DOCKER_USER }} + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Determine image version + id: get_ver + run: | + echo "##[set-output name=version;]$(./bump_version.sh show)" + - name: Build Docker image + run: | + docker build \ + --tag "$IMAGE_NAME" \ + --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ + --build-arg GIT_REMOTE=$(git remote get-url origin) \ + --build-arg VERSION=${{ steps.get_ver.outputs.version }} \ + . + - name: Tag Docker image + run: | + IFS='.' read -r -a version_array \ + <<< "${{ steps.get_ver.outputs.version }}" + docker login --username "$DOCKER_USER" --password "$DOCKER_PW" + docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" + docker tag "$IMAGE_NAME" \ + "${IMAGE_NAME}:${{ steps.get_ver.outputs.version }}" + docker tag "$IMAGE_NAME" \ + "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" + docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version_array[0]}" + - name: Publish image to Docker Hub + run: | + IFS='.' read -r -a version_array \ + <<< "${{ steps.get_ver.outputs.version }}" + docker push "${IMAGE_NAME}:latest" + docker push "${IMAGE_NAME}:${{ steps.get_ver.outputs.version }}" + docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" + docker push "${IMAGE_NAME}:${version_array[0]}" + - name: Publish README.md to Docker Hub + run: ./push_readme.sh From 5686bbee8c2fb1f2a52959d1ec169ff7cbc4ea16 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 12 Nov 2019 11:50:43 -0500 Subject: [PATCH 103/253] Remove Travis configuration. --- .travis.yml | 56 ----------------------------------------------------- 1 file changed, 56 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cedc956..0000000 --- a/.travis.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -dist: xenial -language: python -python: 3.7 -services: docker -env: - global: - - IMAGE_NAME=dhsncats/example - - DOCKER_USER=felddy - # DOCKER_PW - - secure: >- - GWxja6rCcevNRay3uOGEt1wpzGOAoKgfquyror60VtkG07GCJ/rCBa/jNsTRRwwDxYW - MqY6xQlAU+GzNeuBxKAlmyrLRRYZDmcBpPgQQPbiC0b5ftGCzTuhdYAj4Hg+LyHKbNP - H0zsXfQRy2NdWPJ8rr3RFL89a/If0hJVL5OEEHfmzpfvHoDvBJQwhmL2sPZizgCsgh6 - eRStRvUWj3taVb5VZ1uIlet8P3g7azkD7tTNx+wAxhjlLpaQfBcy/CC27ogw+QOvgdF - kefskYz54LLzTyXYhRhACjuGaukpBvQBFzBi3MOFYoLj4kb+9b3mqHlnQZLOSo2zlWV - 30mTC7eed5KE64MuQ6I1E72B0rnAoVpnwAjXVIvzfhKBuzbhS54tWjKbqK6UAGxbg6t - wJDiNaB+nYk6zozJ93TQT02XDUOdDS45hO2Kc/CUwJbrZc7V4aYN6Dj30tWy8d2TI8s - GMPuJm8tyxpT/h2eUUPhVzhMCk+4efIBa6cb99wZgkphLmWQefLLpZy8pMwOH7c3ywU - +6tVPBZxSEm4tFRYyPwgl9qvzE8E0WM3FPNgwVO9aypx61HcO8NVD7alUW2j+s3taHX - aVpaCtsf+bpsXPro3/ofst+FHiFPZGBL2iiSIYtFYI4hkDFq9iGGGexo9zgYS6CMexu - YoZtThDw6kaLI= -cache: - pip: true - directories: - - "$HOME/.cache/pre-commit" -install: - - pip install --upgrade --requirement requirements-test.txt - - version=$(./bump_version.sh show) - - docker build - --tag "$IMAGE_NAME" - --build-arg GIT_COMMIT=$(git log -1 --format=%H) - --build-arg GIT_REMOTE=$(git remote get-url origin) - --build-arg VERSION=${version} - . -script: - - pre-commit run --all-files - - pytest --verbose -before_deploy: - - IFS='.' read -r -a version_array <<< "$version" - - docker login --username "$DOCKER_USER" --password "$DOCKER_PW" - - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" - - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version}" - - docker tag "$IMAGE_NAME" - "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" - - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version_array[0]}" -deploy: - - provider: script - script: docker push "${IMAGE_NAME}:latest" && - docker push "${IMAGE_NAME}:${version}" && - docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" && - docker push "${IMAGE_NAME}:${version_array[0]}" && - ./push_readme.sh - on: - tags: true - python: '3.7' From 1a174ba02813b2301658e10d6017a36256438ac4 Mon Sep 17 00:00:00 2001 From: Felddy Date: Tue, 12 Nov 2019 11:56:56 -0500 Subject: [PATCH 104/253] Create a version lookup step with output. --- .github/workflows/build.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index df36bee..87a4050 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,7 +8,6 @@ env: PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit - jobs: lint: runs-on: ubuntu-latest @@ -65,6 +64,10 @@ jobs: needs: [lint, test] steps: - uses: actions/checkout@v1 + - name: Determine image version + id: get_ver + run: | + echo "##[set-output name=version;]$(./bump_version.sh show)" - name: Build docker image run: | version=$(./bump_version.sh show) @@ -72,14 +75,14 @@ jobs: --tag "$IMAGE_NAME" \ --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${version} \ + --build-arg VERSION=${{ steps.get_ver.outputs.version }} \ . - name: Save docker image run: | mkdir dist version=$(./bump_version.sh show) - docker save $IMAGE_NAME:latest | \ - gzip > dist/"${IMAGE_NAME}"_"${version}".tar.gz + docker save $IMAGE_NAME:latest | gzip > \ + dist/"${IMAGE_NAME}"_"${{ steps.get_ver.outputs.version }}".tar.gz - name: Upload artifacts uses: actions/upload-artifact@v1 with: From 57dfd9ff1e7743552483de99f0302c649851c522 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 14 Nov 2019 11:47:01 -0500 Subject: [PATCH 105/253] Reorder jobs and pass image artifacts. --- .github/workflows/build.yml | 54 ++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 87a4050..1100fd8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,31 +37,8 @@ jobs: pip install --upgrade -r requirements-test.txt - name: Run linters on all files run: pre-commit run --all-files - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Cache pip test requirements - uses: actions/cache@v1 - with: - path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-\ - ${{ hashFiles('**/requirements-test.txt') }}" - restore-keys: | - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade -r requirements-test.txt - - name: Run tests - run: pytest build: runs-on: ubuntu-latest - needs: [lint, test] steps: - uses: actions/checkout@v1 - name: Determine image version @@ -77,7 +54,7 @@ jobs: --build-arg GIT_REMOTE=$(git remote get-url origin) \ --build-arg VERSION=${{ steps.get_ver.outputs.version }} \ . - - name: Save docker image + - name: Save docker image artifact run: | mkdir dist version=$(./bump_version.sh show) @@ -88,3 +65,32 @@ jobs: with: name: dist path: dist + test: + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Cache pip test requirements + uses: actions/cache@v1 + with: + path: ${{ env.PIP_CACHE_DIR }} + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" + restore-keys: | + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade -r requirements-test.txt + - name: Download docker image artifact + uses: actions/download-artifact@v1 + with: + name: dist + - name: Load docker image + run: docker load < dist/*.tar.gz + - name: Run tests + run: pytest From 78348f67d7780dae0b7b3359d654dd3025923f90 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 14 Nov 2019 11:54:46 -0500 Subject: [PATCH 106/253] Simplify artifact naming. --- .github/workflows/build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1100fd8..7c42afb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,8 +58,7 @@ jobs: run: | mkdir dist version=$(./bump_version.sh show) - docker save $IMAGE_NAME:latest | gzip > \ - dist/"${IMAGE_NAME}"_"${{ steps.get_ver.outputs.version }}".tar.gz + docker save $IMAGE_NAME:latest | gzip > dist/image.tar.gz - name: Upload artifacts uses: actions/upload-artifact@v1 with: @@ -91,6 +90,6 @@ jobs: with: name: dist - name: Load docker image - run: docker load < dist/*.tar.gz + run: docker load < dist/image.tar.gz - name: Run tests run: pytest From 9251add910953eb9651268681439b274a4104769 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 14 Nov 2019 12:05:21 -0500 Subject: [PATCH 107/253] Change environment variable name to be CI agnostic. --- tests/container_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/container_test.py b/tests/container_test.py index 50b62d8..524d1d3 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -12,7 +12,7 @@ SECRET_QUOTE = ( "There are no secrets better kept than the secrets everybody guesses." # nosec ) -TRAVIS_TAG = os.getenv("TRAVIS_TAG") +RELEASE_TAG = os.getenv("RELEASE_TAG") VERSION_FILE = "src/version.txt" @@ -54,7 +54,7 @@ def test_output(main_container): @pytest.mark.skipif( - TRAVIS_TAG in [None, ""], reason="this is not a release (TRAVIS_TAG not set)" + RELEASE_TAG in [None, ""], reason="this is not a release (RELEASE_TAG not set)" ) def test_release_version(): """Verify that release tag version agrees with the module version.""" @@ -63,8 +63,8 @@ def test_release_version(): exec(f.read(), pkg_vars) # nosec project_version = pkg_vars["__version__"] assert ( - TRAVIS_TAG == f"v{project_version}" - ), "TRAVIS_TAG does not match the project version" + RELEASE_TAG == f"v{project_version}" + ), "RELEASE_TAG does not match the project version" def test_log_version(version_container): From 7ed5c76f8b249e04f7c9b8b7ec684fad395a9c12 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 14 Nov 2019 12:19:44 -0500 Subject: [PATCH 108/253] Move images from dhsncats to cisagov docker hub org. --- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 2 +- README.md | 10 +++++----- docker-compose.yml | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c42afb..1eddebe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ name: build on: [push] env: - IMAGE_NAME: dhsncats/example + IMAGE_NAME: cisagov/example PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8fcc8e1..6fe7223 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,7 @@ on: types: [prereleased, released] env: - IMAGE_NAME: dhsncats/example + IMAGE_NAME: cisagov/example DOCKER_PW: ${{ secrets.DOCKER_PW }} DOCKER_USER: ${{ secrets.DOCKER_USER }} diff --git a/README.md b/README.md index 5b2da6b..5b39478 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ ## Docker Image ## -![MicroBadger Layers](https://img.shields.io/microbadger/layers/dhsncats/example.svg) -![MicroBadger Size](https://img.shields.io/microbadger/image-size/dhsncats/example.svg) +![MicroBadger Layers](https://img.shields.io/microbadger/layers/cisagov/example.svg) +![MicroBadger Size](https://img.shields.io/microbadger/image-size/cisagov/example.svg) This is a docker skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub docker project @@ -20,11 +20,11 @@ appropriate for docker containers and the major languages that we use. ### Install ### -Pull `dhsncats/example` from the Docker repository: +Pull `cisagov/example` from the Docker repository: - docker pull dhsncats/example + docker pull cisagov/example -Or build `dhsncats/example` from source: +Or build `cisagov/example` from source: git clone https://github.com/cisagov/skeleton-docker.git cd skeleton-docker diff --git a/docker-compose.yml b/docker-compose.yml index f9a54ad..6774387 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: # e.g., --build-arg VERSION=0.0.1 context: . dockerfile: Dockerfile - image: dhsncats/example + image: cisagov/example init: true restart: "no" environment: @@ -31,7 +31,7 @@ services: example-version: # Run the container to collect version information - image: dhsncats/example + image: cisagov/example init: true restart: "no" command: --version From e97faf6bd11ecf8753cf757e1cf182c77b04bfdc Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 14 Nov 2019 14:15:35 -0500 Subject: [PATCH 109/253] Pass in the github release tag for testing. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1eddebe..9643f76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,4 +92,6 @@ jobs: - name: Load docker image run: docker load < dist/image.tar.gz - name: Run tests + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} run: pytest From 7c0b30b495e689b84317b390eff36425d7c70cdd Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 15 Nov 2019 11:50:38 -0500 Subject: [PATCH 110/253] Force pre-commit hooks to use python3. Fixes #23 --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d7bcbc..7626c79 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,8 @@ --- +default_language_version: + # force all unspecified python hooks to run python3 + python: python3 + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 From 30a16a38420cca91cda0cd2543484aa4a2e125cc Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 15 Nov 2019 11:51:46 -0500 Subject: [PATCH 111/253] Autoupdate pre-commit hooks. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7626c79..27477d3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a3 + rev: v4.1.1a5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml @@ -75,6 +75,6 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.18.2 + rev: 1.19.1 hooks: - id: prettier From e39dc8881915f36fdb70c9c58f0278700523b294 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 18 Nov 2019 10:33:28 -0500 Subject: [PATCH 112/253] Add codeowners file with team OIS maintainers. See: https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners --- .github/CODEOWNERS | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..ca9c8dc --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# these owners will be requested for review when someone +# opens a pull request. +* @dav3r @felddy @jsf9k @mcdonnnj From b89eecb788084c95930554679cb38b32548c3c8d Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Mon, 18 Nov 2019 11:11:06 -0500 Subject: [PATCH 113/253] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ca9c8dc..32918e8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ # the repo. Unless a later match takes precedence, # these owners will be requested for review when someone # opens a pull request. -* @dav3r @felddy @jsf9k @mcdonnnj +* @dav3r @felddy @jsf9k @mcdonnnj @cisagov/team-ois From 18b5d12c784dad59c461af36922cc2082f1dcffd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 19 Dec 2019 15:32:36 -0500 Subject: [PATCH 114/253] Add seed-isort-config and isort to pre-commit hooks. Add isort configuration file. --- .isort.cfg | 12 ++++++++++++ .pre-commit-config.yaml | 10 ++++++++++ 2 files changed, 22 insertions(+) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..5216e47 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,12 @@ +[settings] +combine_star=true +force_sort_within_sections=true + +import_heading_stdlib=Standard Python Libraries +import_heading_thirdparty=Third-Party Libraries +import_heading_firstparty=cisagov Libraries + +# Should be auto-populated by seed-isort-config hook +known_third_party= +# These must be manually set to correctly separate them from third party libraries +known_first_party= diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27477d3..c8fc88a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,6 +60,16 @@ repos: rev: 19.10b0 hooks: - id: black + - repo: https://github.com/asottile/seed-isort-config + rev: v1.9.3 + hooks: + - id: seed-isort-config + - repo: https://github.com/pre-commit/mirrors-isort + # pick the isort version you'd like to use from + # https://github.com/pre-commit/mirrors-isort/releases + rev: v4.3.21 + hooks: + - id: isort - repo: https://github.com/ansible/ansible-lint.git rev: v4.1.1a5 hooks: From 2fa4cbe194ac05ff89e377ad91adfa3fc7eddb6f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 24 Jan 2020 16:45:48 -0500 Subject: [PATCH 115/253] Make workflow run when a PR is opened, synchronized, or reopened A user forked cisagov/scan-target-data and created a pull request, but the required GitHub Action(s) did not run. This is presumably because the user does not have Actions enabled in his or her fork. Ideally, the required Action(s) would run in cisagov/scan-target-data when a PR to merge changes back is created. Based on my reading of this link, adding the "pull_request" event type should make this happen: https://help.github.com/en/actions/automating-your-workflow-with-github-actions/events-that-trigger-workflows#pull-request-events-for-forked-repositories --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d12843d..c9cc05b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,7 @@ --- name: build -on: [push] +on: [push, pull_request] jobs: build: From dbd589d2e27b517726fec167987058b3d687992f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 24 Jan 2020 17:10:49 -0500 Subject: [PATCH 116/253] Improve list formatting --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c9cc05b..e6c14e6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,7 +1,10 @@ --- name: build -on: [push, pull_request] +on: [ + push, + pull_request +] jobs: build: From deb186c98b5539654402c2c252bc3d55d2190aeb Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 27 Jan 2020 11:14:04 -0500 Subject: [PATCH 117/253] Add changes made by isort pre-commit hook --- .isort.cfg | 2 +- tests/conftest.py | 2 +- tests/container_test.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 5216e47..3b2f682 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -7,6 +7,6 @@ import_heading_thirdparty=Third-Party Libraries import_heading_firstparty=cisagov Libraries # Should be auto-populated by seed-isort-config hook -known_third_party= +known_third_party=pytest # These must be manually set to correctly separate them from third party libraries known_first_party= diff --git a/tests/conftest.py b/tests/conftest.py index d62dff6..28d6c42 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,9 @@ https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins """ +# Third-Party Libraries import pytest - MAIN_SERVICE_NAME = "example" VERSION_SERVICE_NAME = f"{MAIN_SERVICE_NAME}-version" diff --git a/tests/container_test.py b/tests/container_test.py index 524d1d3..90ea1a3 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,9 +1,11 @@ #!/usr/bin/env pytest -vs """Tests for example container.""" +# Standard Python Libraries import os import time +# Third-Party Libraries import pytest ENV_VAR = "ECHO_MESSAGE" From 532751627a347637814e8a7f626dad487a1b6053 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 10 Feb 2020 10:39:07 -0500 Subject: [PATCH 118/253] Backported changes to CONTRIBUTING.md from the development guide. --- CONTRIBUTING.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93addc2..eb00ca9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,10 +56,31 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux (or on the Mac, if you don't want to use `brew`) you can use +For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you +don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to -install the necessary tools. When you are finished you will need to -add the same two lines above to your profile. +install the necessary tools. Before running this ensure that you have +installed the prerequisites for your platform according to the +[`pyenv` wiki +page](https://github.com/pyenv/pyenv/wiki/common-build-problems). + +On WSL you should treat your platform as whatever Linux distribution +you've chosen to install. + +Once you have installed `pyenv` you will need to add the following +lines to your `.bashrc`: + +```bash +export PATH="$PATH:$HOME/.pyenv/bin" +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" +``` + +If you are using a shell other than `bash` you should follow the +instructions that the `pyenv-installer` script outputs. + +You will need to reload your shell for these changes to take effect so +you can begin to use `pyenv`. For a list of Python versions that are already installed and ready to use with `pyenv`, use the command `pyenv versions`. To see a list of From f7a4166ad67d961324bc44130e092eb1ddebd320 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 11 Feb 2020 10:41:17 -0500 Subject: [PATCH 119/253] Update Python version used to 3.8 Update actions/checkout to v2 Update formatting to match downstream children --- .github/workflows/build.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6c14e6..4953f7c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,20 +10,16 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 with: - python-version: 3.7 - + python-version: 3.8 - name: Cache pre-commit hooks uses: actions/cache@v1 with: path: ~/.cache/pre-commit key: "${{ runner.os }}-pre-commit-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - - name: Cache pip test requirements uses: actions/cache@v1 with: @@ -33,11 +29,9 @@ jobs: restore-keys: | ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- - - name: Install dependencies run: | python -m pip install --upgrade pip pip install --upgrade -r requirements-test.txt - - name: Run pre-commit on all files run: pre-commit run --all-files From b857939b3d0c5393b8b5528a6bb9bce2ff0f736f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 00:01:16 -0500 Subject: [PATCH 120/253] Run pre-commit autoupdate. --- .pre-commit-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c8fc88a..7856658 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v2.5.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -27,13 +27,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.19.0 + rev: v0.22.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.18.0 + rev: v1.20.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.25.1 + rev: v1.26.2 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v1.9.3 + rev: v1.9.4 hooks: - id: seed-isort-config - repo: https://github.com/pre-commit/mirrors-isort @@ -71,7 +71,7 @@ repos: hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a5 + rev: v4.2.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml @@ -81,7 +81,7 @@ repos: - id: terraform_fmt - id: terraform_validate_no_variables - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v1.0.0 + rev: v1.0.1 hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier From d99fd00bc2e5c4a0afeb2d6717dac7fe77f64d33 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 00:14:11 -0500 Subject: [PATCH 121/253] Flip cache order to mirror how it is done downstream. --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4953f7c..76801a8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,12 +14,6 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.8 - - name: Cache pre-commit hooks - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: "${{ runner.os }}-pre-commit-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Cache pip test requirements uses: actions/cache@v1 with: @@ -29,6 +23,12 @@ jobs: restore-keys: | ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- + - name: Cache pre-commit hooks + uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: "${{ runner.os }}-pre-commit-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Install dependencies run: | python -m pip install --upgrade pip From bf7f97bd480df8dfc0e7de063173cd4ccc9e8827 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 16:36:17 -0500 Subject: [PATCH 122/253] Add missing space in .bandit.yml --- .bandit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bandit.yml b/.bandit.yml index 5ae161d..2b618f6 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -10,4 +10,4 @@ tests: # - B102 skips: - - B101 # skip "assert used" check since assertions are required in pytests + - B101 # skip "assert used" check since assertions are required in pytests From e96577bce4b3b6aefa044943e478301a7d11288f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 18 Feb 2020 18:04:06 -0500 Subject: [PATCH 123/253] All references to '-r' for pip calls have been replaced with the more verbose '--requirement'. --- .github/workflows/build.yml | 2 +- CONTRIBUTING.md | 2 +- requirements-dev.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 76801a8..aff7e7a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,6 +32,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade -r requirements-test.txt + pip install --upgrade --requirement requirements-test.txt - name: Run pre-commit on all files run: pre-commit run --all-files diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eb00ca9..dacaaad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,7 +102,7 @@ commands: cd skeleton-generic pyenv virtualenv skeleton-generic pyenv local skeleton-generic -pip install -r requirements-dev.txt +pip install --requirement requirements-dev.txt ``` #### Installing the pre-commit hook #### diff --git a/requirements-dev.txt b/requirements-dev.txt index f122cc5..d84ee68 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ --r requirements-test.txt +--requirement requirements-test.txt ipython From 067ee0850c154845b7de623988c5a1bd5ce67d3a Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Feb 2020 17:29:09 -0500 Subject: [PATCH 124/253] Autoupdate pre-commit hooks. Add mypy. --- .pre-commit-config.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7856658..46cea9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.26.2 + rev: v2.0.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -74,7 +74,7 @@ repos: rev: v4.2.0 hooks: - id: ansible-lint - # files: molecule/default/playbook.yml + # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: @@ -88,3 +88,7 @@ repos: rev: 1.19.1 hooks: - id: prettier + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.761 + hooks: + - id: mypy From bf366086f48d9b7a7e49b9f44f33b44f6a23aeb3 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 2 Mar 2020 08:11:32 -0500 Subject: [PATCH 125/253] Add .mypy_cache to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 073a081..95b74cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +.mypy_cache __pycache__ .python-version From 454864bc029ce968cbeb8f9f5b88ad1c54522335 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 3 Mar 2020 16:49:53 -0500 Subject: [PATCH 126/253] Incorporate the Python version into keys for pip and pre-commit caches. This should resolve the issue seen when the Python version changes before there is an update to .pre-commit-config.yml which results in pre-commit pointing to a non-existent Python installation. --- .github/workflows/build.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aff7e7a..a92cd83 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,20 +14,25 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.8 + - name: Store installed Python version + run: | + echo "::set-env name=PY_VERSION::"\ + "$(python -c "import platform;print(platform.python_version())")" - name: Cache pip test requirements uses: actions/cache@v1 with: path: ~/.cache/pip - key: "${{ runner.os }}-pip-test-\ + key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | + ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}- ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- - name: Cache pre-commit hooks uses: actions/cache@v1 with: path: ~/.cache/pre-commit - key: "${{ runner.os }}-pre-commit-\ + key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Install dependencies run: | From 478f1688a108dd0f6e634c5f03c7ba6cb816e3f2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 4 Mar 2020 08:41:59 -0500 Subject: [PATCH 127/253] Change the cache paths from hardcoded values in their appropriate blocks to environment variables declared before the job block. --- .github/workflows/build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a92cd83..298e93c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,10 @@ on: [ pull_request ] +env: + PIP_CACHE_DIR: ~/.cache/pip + PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + jobs: build: runs-on: ubuntu-latest @@ -21,7 +25,7 @@ jobs: - name: Cache pip test requirements uses: actions/cache@v1 with: - path: ~/.cache/pip + path: ${{ env.PIP_CACHE_DIR }} key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | @@ -31,7 +35,7 @@ jobs: - name: Cache pre-commit hooks uses: actions/cache@v1 with: - path: ~/.cache/pre-commit + path: ${{ env.PRE_COMMIT_CACHE_DIR }} key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Install dependencies From d7913343523e9841e1d295a5203e2043f055064b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 4 Mar 2020 10:56:39 -0500 Subject: [PATCH 128/253] Add a rule for markdownlint to allow multiple headers with the same name as long as they are not nested in the same heading group. --- .mdl_config.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.mdl_config.json b/.mdl_config.json index 492955a..7a6f3f8 100644 --- a/.mdl_config.json +++ b/.mdl_config.json @@ -3,5 +3,8 @@ "code_blocks": false, "tables": false }, + "MD024": { + "allow_different_nesting": true + }, "default": true } From 61790a9bbd22cede3cd41d94115d0e8ad8ba51d1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 31 Mar 2020 14:41:10 -0400 Subject: [PATCH 129/253] Fixed broken hook id for pre-commit-terraform: terraform_validate_no_variables was changed to terraform_validate in the following commit: https://github.com/antonbabenko/pre-commit-terraform/commit/35e0356188b64a4c5af9a4e7200d936e514cba71. Ran pre-commit autoupdate. --- .pre-commit-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46cea9e..42b824f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.20.0 + rev: v1.21.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.0.0 + rev: v2.1.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v1.9.4 + rev: v2.1.0 hooks: - id: seed-isort-config - repo: https://github.com/pre-commit/mirrors-isort @@ -76,19 +76,19 @@ repos: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.12.0 + rev: v1.27.0 hooks: - id: terraform_fmt - - id: terraform_validate_no_variables + - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.1 hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 1.19.1 + rev: 2.0.2 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.761 + rev: v0.770 hooks: - id: mypy From 0bc1aab45e2338e658a63a9b92e9f9d14eafdec8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 12:05:57 -0400 Subject: [PATCH 130/253] Ran `pre-commit autoupdate` to get the latest version of the `pre-commit-terraform` hook since the PR I submitted, https://github.com/antonbabenko/pre-commit-terraform/pull/100, was approved. This will fix issues with `skeleton-tf-module` related to multiple directories with Terraform code. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 42b824f..129aa39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,7 +76,7 @@ repos: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.27.0 + rev: v1.29.0 hooks: - id: terraform_fmt - id: terraform_validate @@ -85,7 +85,7 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.0.2 + rev: 2.0.4 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From b01a0ee7bd1e0854b4c4c6bb999316c21221c97f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 13:47:18 -0400 Subject: [PATCH 131/253] Explicitly install pre-commit hooks as its own step so it's clearer when the failure is with setup rather than hooks running. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 298e93c..6026d47 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,5 +42,7 @@ jobs: run: | python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt + - name: Install pre-commit hooks + run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files From 1f3d440a1af6283621b5584da9b56e2747d40a0b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 19:02:31 -0400 Subject: [PATCH 132/253] Rename action to accurately reflect that we are only linting, not building. Adjust description for pre-commit hook setup to better convey what is being done. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6026d47..9c6b03a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ env: PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit jobs: - build: + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -42,7 +42,7 @@ jobs: run: | python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt - - name: Install pre-commit hooks + - name: Set up pre-commit hook environments run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files From 647b351997ab7cafd391bdabda7b1b5e68214b8a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 6 Apr 2020 19:04:21 -0400 Subject: [PATCH 133/253] Sort .gitignore entries. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 95b74cd..e00826d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ .mypy_cache -__pycache__ .python-version +__pycache__ From c8b4bcf126989abf35e4ce226263ef7da139c959 Mon Sep 17 00:00:00 2001 From: Hillary Date: Tue, 14 Apr 2020 12:43:18 -0400 Subject: [PATCH 134/253] Update CODEOWNERS --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 32918e8..a3619be 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ # the repo. Unless a later match takes precedence, # these owners will be requested for review when someone # opens a pull request. -* @dav3r @felddy @jsf9k @mcdonnnj @cisagov/team-ois +* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj @cisagov/team-ois From 8c99c5294a798bca98a68212507fce924505a289 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 30 Apr 2020 23:31:59 -0400 Subject: [PATCH 135/253] Allow events from apb to rebuild this repository weekly. --- .github/workflows/build.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9c6b03a..5fe2f8b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,11 @@ --- name: build -on: [ - push, - pull_request -] +on: + push: + pull_request: + repository_dispatch: + types: [apb] env: PIP_CACHE_DIR: ~/.cache/pip From 7d7a6774571c4727b4858b1117c31f97160131da Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 8 May 2020 17:49:17 -0400 Subject: [PATCH 136/253] Updated pre-commit configuration with 'pre-commit autoupdate'. --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 129aa39..de0152b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,13 +27,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.22.0 + rev: v0.23.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.21.0 + rev: v1.23.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -41,13 +41,13 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.7.9 + rev: 3.8.0a2 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.1.0 + rev: v2.4.1 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v2.1.0 + rev: v2.1.1 hooks: - id: seed-isort-config - repo: https://github.com/pre-commit/mirrors-isort @@ -71,12 +71,12 @@ repos: hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.2.0 + rev: v4.3.0a0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.29.0 + rev: v1.30.0 hooks: - id: terraform_fmt - id: terraform_validate @@ -85,7 +85,7 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.0.4 + rev: 2.0.5 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From cda200cfd1be3dfa44fbb2f271e571b43d560691 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 11 May 2020 21:52:31 -0400 Subject: [PATCH 137/253] Add Lineage configuration. --- .github/lineage.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/lineage.yml diff --git a/.github/lineage.yml b/.github/lineage.yml new file mode 100644 index 0000000..8dfc20b --- /dev/null +++ b/.github/lineage.yml @@ -0,0 +1,6 @@ +--- +version: "1" + +lineage: + skeleton: + remote-url: https://github.com/cisagov/skeleton-generic.git From ce6658abc69d1444e51f45c20441c98e6b54c086 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 15:25:31 -0400 Subject: [PATCH 138/253] Update isort pre-commit hook source repository. The mirrors-isort respository has been deprecated per this Pull Request: https://github.com/pre-commit/mirrors-isort/pull/13 Since isort includes pre-commit configuration in current versions we will switch to using it directly. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de0152b..5be35e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,10 +64,10 @@ repos: rev: v2.1.1 hooks: - id: seed-isort-config - - repo: https://github.com/pre-commit/mirrors-isort + - repo: https://github.com/timothycrosley/isort # pick the isort version you'd like to use from # https://github.com/pre-commit/mirrors-isort/releases - rev: v4.3.21 + rev: 4.3.21 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git From daaebc1cc13782e8fb39be6815aa53bf3efd44f3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 15:30:04 -0400 Subject: [PATCH 139/253] Ran pre-commit autoupdate. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5be35e4..a2e4f76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.0a2 + rev: 3.8.1 hooks: - id: flake8 additional_dependencies: From 2399f24622b1d6be92c494653d9308e9694c5496 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 15:35:36 -0400 Subject: [PATCH 140/253] Remove legacy comment from the isort hook declaration. This comment references the mirrors-isort repository that we are no longer using. --- .pre-commit-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2e4f76..bf8ae6e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,8 +65,6 @@ repos: hooks: - id: seed-isort-config - repo: https://github.com/timothycrosley/isort - # pick the isort version you'd like to use from - # https://github.com/pre-commit/mirrors-isort/releases rev: 4.3.21 hooks: - id: isort From fe4d5eb72a8f6ce9e23daae98d7a0c87675a9da4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 16:07:18 -0400 Subject: [PATCH 141/253] Disable the terraform_validate hook. We have seen a number of issues related to this hook ever since it was re-enabled. It will need to remain disabled until at least the 0.13 Terraform release, and can only be re-enabled if all issues we have seen have been resolved in how `terraform validate` operates. --- .pre-commit-config.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf8ae6e..74af27c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -77,7 +77,19 @@ repos: rev: v1.30.0 hooks: - id: terraform_fmt - - id: terraform_validate + # There are ongoing issues with how this command works. This issue + # documents the core issue: + # https://github.com/hashicorp/terraform/issues/21408 + # We have seen issues primarily with proxy providers and Terraform code + # that uses remote state. The PR + # https://github.com/hashicorp/terraform/pull/24887 + # has been approved and is part of the 0.13 release to resolve the issue + # with remote states. + # The PR + # https://github.com/hashicorp/terraform/pull/24896 + # is a proprosed fix to deal with `terraform validate` with proxy + # providers (among other configurations). + # - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.1 hooks: From f831ef827cae16fd288f2e7a16fedc53c6e06873 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 May 2020 21:34:58 -0400 Subject: [PATCH 142/253] Add final statement to descriptionf or why terraform_validate is disabled. Review noticed that there lacked a determination for what we were doing about the problem with the terraform_validate hook. I described the problems but failed to mention what our path forward would be. This commit rectifies that oversight. --- .pre-commit-config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74af27c..b7c5518 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -89,6 +89,9 @@ repos: # https://github.com/hashicorp/terraform/pull/24896 # is a proprosed fix to deal with `terraform validate` with proxy # providers (among other configurations). + # We have decided to disable the terraform_validate hook until the issues + # above have been resolved, which we hope will be with the release of + # Terraform 0.13. # - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit rev: v1.0.1 From 87ea53022c39f3adead2688fbae5698147b279d8 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 9 Jun 2020 15:10:11 -0400 Subject: [PATCH 143/253] Add setuptools and wheel as pip dependencies setuptools usually comes along with pip, but wheel does not. Using wheel where possible to build python extensions is more modern and more security conscious than using setup.py. --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0a8547b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +setuptools +wheel From 7f8d430a309b22a3410949e64ccd60ad5a1dc79f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 9 Jun 2020 15:19:05 -0400 Subject: [PATCH 144/253] Pull in requirements.txt from requirements-test.txt --- requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-test.txt b/requirements-test.txt index 416634f..66f74db 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1,2 @@ +--requirement requirements.txt pre-commit From af06915b05674695cb5d9bd8418c6b37f77ec3bc Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 14:57:04 -0400 Subject: [PATCH 145/253] Add tool and docs to automate development environment setup. --- CONTRIBUTING.md | 11 +++ setup-env | 175 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100755 setup-env diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dacaaad..2bd2451 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,17 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. +If you already have `pyenv` and `pyenv-virtualenv` configured you can +take advantage of the `setup-env` tool in this repo to automate the +entire environment configuration process. + +```bash +./setup-env +``` + +Otherwise, follow the steps below to manually configure your +environment. + #### Installing and using `pyenv` and `pyenv-virtualenv` #### On the Mac, installation is as simple as `brew install pyenv diff --git a/setup-env b/setup-env new file mode 100755 index 0000000..65c5b6f --- /dev/null +++ b/setup-env @@ -0,0 +1,175 @@ +#!/usr/bin/env bash + +set -o nounset +set -o errexit +set -o pipefail + +USAGE=$(cat << 'END_OF_LINE' +This script is used to configure a developement environment for this repo. + +It does the following: + - Verifies pyenv and pyenv-virtualenv are installed. + - Creates a Python virtual environment. + - Configures the activation of the virtual enviroment for the repo directory. + - Installs the requirements required for development. + - Installs git pre-commit hooks. + - Configures git upstream remote "lineage" repositories. + +usage: setup-env [--force] [--help] [virt_env_name] + +END_OF_LINE +) + +# Flag to force deletion and creation of virtual environment +FORCE=0 + +# Positional parameters +PARAMS="" + +# Parse command line arguments +while (( "$#" )); do + case "$1" in + -f|--force) + FORCE=1 + shift + ;; + -h|--help) + echo "${USAGE}" + exit 0 + ;; + -*) # unsupported flags + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; + *) # preserve positional arguments + PARAMS="$PARAMS $1" + shift + ;; + esac +done + +# set positional arguments in their proper place +eval set -- "$PARAMS" + +# Check to see if pyenv is installed +if [ -z "$(which pyenv)" ] || [ -z "$(which pyenv-virtualenv)" ]; then + echo "pyenv and pyenv-virtualenv are required." + if [[ "$OSTYPE" == "darwin"* ]]; then + cat << 'END_OF_LINE' + + On the Mac, installation is as simple as "brew install pyenv + pyenv-virtualenv" and adding this to your profile: + + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + +END_OF_LINE + + fi + cat << 'END_OF_LINE' + For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + to use "brew") you can use https://github.com/pyenv/pyenv-installer to install + the necessary tools. Before running this ensure that you have installed the + prerequisites for your platform according to the pyenv wiki page, + https://github.com/pyenv/pyenv/wiki/common-build-problems. + + On WSL you should treat your platform as whatever Linux distribution you've + chosen to install. + + Once you have installed "pyenv" you will need to add the following lines to + your ".bashrc": + + export PATH="$PATH:$HOME/.pyenv/bin" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" +END_OF_LINE + exit 1 +fi + +set +o nounset +# Determine the virtual environment name +if [ "$1" ]; then + # Use the user-provided environment name + env_name=$1 +else + # Set the environment name to the last part of the working directory. + env_name=${PWD##*/} +fi +set -o nounset + +# Remove any lingering local configuration. +if [ $FORCE -ne 0 ]; then + rm -f .python-version + pyenv virtualenv-delete --force "${env_name}" || true +elif [[ -f .python-version ]]; then + cat << 'END_OF_LINE' + An existing .python-version file was found. Either remove this file yourself + or re-run with --force option to have it deleted along with the associated + virtual environment. + + rm .python-version + +END_OF_LINE + exit 1 +fi + +# Create a new virutal environment for this project +if ! pyenv virtualenv "${env_name}"; then + cat << END_OF_LINE + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. + + pyenv virtualenv-delete ${env_name} + +END_OF_LINE + exit 1 +fi + +# Activate the new virtual environment +pyenv local "${env_name}" + +# Upgrade pip and friends +python -m pip install --upgrade pip setuptools wheel + +# Find a requirements file (if possible) and install +for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do + if [[ -f $req_file ]] + then + pip install -r $req_file + break + fi +done + +# Install git pre-commit hooks +pre-commit install + +# Setup git remotes from lineage configuration +# This could fail if the remotes are already setup, but that is ok. +set +o errexit + +eval "$(python3 << 'END_OF_LINE' +from pathlib import Path +import yaml +import sys + +LINEAGE_CONFIG = Path(".github/lineage.yml") + +if not LINEAGE_CONFIG.exists(): + print('No lineage configuration found.', file=sys.stderr) + sys.exit(0) + +with LINEAGE_CONFIG.open("r") as f: + lineage = yaml.safe_load(stream=f) + +if lineage["version"] == "1": + for parent_name, v in lineage["lineage"].items(): + remote_url = v["remote-url"] + print(f"git remote add {parent_name} {remote_url};") + print(f"git remote set-url --push {parent_name} no_push;") +else: + print(f'Unsupported lineage version: {lineage["version"]}', file=sys.stderr) +END_OF_LINE +)" + +# Qapla +echo "Success!" From c50094a8dee2573462c355e82e01eba13fb1fb14 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 15:15:18 -0400 Subject: [PATCH 146/253] Fix typo. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 65c5b6f..5015beb 100755 --- a/setup-env +++ b/setup-env @@ -113,7 +113,7 @@ END_OF_LINE exit 1 fi -# Create a new virutal environment for this project +# Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this From 3c2781c8cc3d87d4e1433100e243679e899d7fe1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 15:25:59 -0400 Subject: [PATCH 147/253] Correct syntax on fence. Closes CVE-2020-3.14159 --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bd2451..8635fea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,7 @@ If you already have `pyenv` and `pyenv-virtualenv` configured you can take advantage of the `setup-env` tool in this repo to automate the entire environment configuration process. -```bash +```console ./setup-env ``` From 23ef29c8a301a6851664ae0ccd8633363f95648f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:30:46 -0400 Subject: [PATCH 148/253] Clarify comment with text from `pyenv local --help` --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5015beb..5f404b3 100755 --- a/setup-env +++ b/setup-env @@ -125,7 +125,8 @@ END_OF_LINE exit 1 fi -# Activate the new virtual environment +# Set the local application-specific Python version(s) by writing the +# version name to a file named `.python-version'. pyenv local "${env_name}" # Upgrade pip and friends From 5d1d134c88aa203d69d4baf88e9b1e5d4b7a2eab Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:34:53 -0400 Subject: [PATCH 149/253] Change to python3 to standardize how we call python. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5f404b3..a767b3d 100755 --- a/setup-env +++ b/setup-env @@ -130,7 +130,7 @@ fi pyenv local "${env_name}" # Upgrade pip and friends -python -m pip install --upgrade pip setuptools wheel +python3 -m pip install --upgrade pip setuptools wheel # Find a requirements file (if possible) and install for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do From 8a105ca04010ca92cd066a525b840fe1c6de66f5 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:50:42 -0400 Subject: [PATCH 150/253] Respect customs. --- setup-env | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index a767b3d..713687a 100755 --- a/setup-env +++ b/setup-env @@ -134,9 +134,8 @@ python3 -m pip install --upgrade pip setuptools wheel # Find a requirements file (if possible) and install for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt"; do - if [[ -f $req_file ]] - then - pip install -r $req_file + if [[ -f $req_file ]]; then + pip install --requirement $req_file break fi done From ae21af108405bde6ea772d804f71ecf3cd677895 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:55:01 -0400 Subject: [PATCH 151/253] Improve usage message. --- setup-env | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 713687a..7bc99f9 100755 --- a/setup-env +++ b/setup-env @@ -5,17 +5,25 @@ set -o errexit set -o pipefail USAGE=$(cat << 'END_OF_LINE' -This script is used to configure a developement environment for this repo. +Configure a developement environment for this repository. It does the following: - Verifies pyenv and pyenv-virtualenv are installed. - Creates a Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - - Installs the requirements required for development. + - Installs the requirements needed for development. - Installs git pre-commit hooks. - Configures git upstream remote "lineage" repositories. -usage: setup-env [--force] [--help] [virt_env_name] +Usage: + setup-env [--force] [virt_env_name] + setup-env (-h | --help) + +Options: + -f --force Delete virtual enviroment if it already exists. + -h --help Show this message. + -i --install-hooks Install hook environments for all environments in the + pre-commit config file. END_OF_LINE ) From c5e7edf46b736965216b258ac562cbf6ad6e1218 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 16:55:34 -0400 Subject: [PATCH 152/253] Add flag to install pre-commit hooks now. --- setup-env | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 7bc99f9..5b6dacd 100755 --- a/setup-env +++ b/setup-env @@ -37,6 +37,10 @@ PARAMS="" # Parse command line arguments while (( "$#" )); do case "$1" in + -i|--install-hooks) + INSTALL_HOOKS=1 + shift + ;; -f|--force) FORCE=1 shift @@ -148,8 +152,8 @@ for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt fi done -# Install git pre-commit hooks -pre-commit install +# Install git pre-commit hooks now or later. +pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} # Setup git remotes from lineage configuration # This could fail if the remotes are already setup, but that is ok. From 482a370679570e55233c13ef6df644ee3437694e Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:02:03 -0400 Subject: [PATCH 153/253] Switch to a POSIX compliant command... command. This is the way. I learned something today. https://unix.stackexchange.com/questions/85249/why-not-use-which-what-to-use-then --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5b6dacd..9d1ed6a 100755 --- a/setup-env +++ b/setup-env @@ -64,7 +64,7 @@ done eval set -- "$PARAMS" # Check to see if pyenv is installed -if [ -z "$(which pyenv)" ] || [ -z "$(which pyenv-virtualenv)" ]; then +if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then echo "pyenv and pyenv-virtualenv are required." if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' From 466ef0cce560fc18b4d657caef943f9c07626fc8 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:05:19 -0400 Subject: [PATCH 154/253] Sort command line flag case statement. --- setup-env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup-env b/setup-env index 9d1ed6a..0b44a24 100755 --- a/setup-env +++ b/setup-env @@ -37,10 +37,6 @@ PARAMS="" # Parse command line arguments while (( "$#" )); do case "$1" in - -i|--install-hooks) - INSTALL_HOOKS=1 - shift - ;; -f|--force) FORCE=1 shift @@ -49,6 +45,10 @@ while (( "$#" )); do echo "${USAGE}" exit 0 ;; + -i|--install-hooks) + INSTALL_HOOKS=1 + shift + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 From 996ae2677a33e1631c8ac80a6e9c682be22e6ac9 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:13:51 -0400 Subject: [PATCH 155/253] Add recommendation to install brew, instead of assuming it is available. --- CONTRIBUTING.md | 5 +++-- setup-env | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8635fea..6434d30 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,8 +59,9 @@ environment. #### Installing and using `pyenv` and `pyenv-virtualenv` #### -On the Mac, installation is as simple as `brew install pyenv -pyenv-virtualenv` and adding this to your profile: +On the Mac, we recommend installing [brew](https://brew.sh/). Then +installation is as simple as `brew install pyenv pyenv-virtualenv` and +adding this to your profile: ```bash eval "$(pyenv init -)" diff --git a/setup-env b/setup-env index 0b44a24..46620e0 100755 --- a/setup-env +++ b/setup-env @@ -69,8 +69,9 @@ if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' - On the Mac, installation is as simple as "brew install pyenv - pyenv-virtualenv" and adding this to your profile: + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your + profile: eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 31980ed4a8624baa682735d0ecfef611582e55a1 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:18:19 -0400 Subject: [PATCH 156/253] Clean up usage syntax since we now have multiple options. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 46620e0..57abe56 100755 --- a/setup-env +++ b/setup-env @@ -16,7 +16,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [--force] [virt_env_name] + setup-env [options] [virt_env_name] setup-env (-h | --help) Options: From db362b912512418280e8b74891de812016e4fb9f Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 22 Jun 2020 17:37:34 -0400 Subject: [PATCH 157/253] Normalize quotes. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 57abe56..4d822c4 100755 --- a/setup-env +++ b/setup-env @@ -168,7 +168,7 @@ import sys LINEAGE_CONFIG = Path(".github/lineage.yml") if not LINEAGE_CONFIG.exists(): - print('No lineage configuration found.', file=sys.stderr) + print("No lineage configuration found.", file=sys.stderr) sys.exit(0) with LINEAGE_CONFIG.open("r") as f: From 91279d3ac2a17383acbea2f82a193b3fefdaa895 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 10 Jul 2020 21:28:58 -0400 Subject: [PATCH 158/253] Perform a pre-commit autoupdate for 2020-07 Notable version changes: - docker-pre-commit v2.0.0 This release includes the addition of two new hooks: hadolint and hadolint-system. These allow for linting Dockerfiles with the hadolint Docker image or a system installation of hadolint respectively. - isort 5.0.7 This release takes us from isort 4 to isort 5. There are a multitude of changes with information available on the isort 5 release page at https://timothycrosley.github.io/isort/docs/major_releases/introducing_isort_5/ The most notable feature for us is the addition of built-in profiles for isort to run under. These profiles provide settings for conforming to various popular styles, and black is one of the profiles. --- .pre-commit-config.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7c5518..52b6473 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v3.1.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -27,7 +27,7 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.23.0 + rev: v0.23.2 hooks: - id: markdownlint args: @@ -41,13 +41,13 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.1 + rev: 3.8.3 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.4.1 + rev: v2.7.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,20 +61,20 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v2.1.1 + rev: v2.2.0 hooks: - id: seed-isort-config - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + rev: 5.0.7 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.3.0a0 + rev: v4.3.0a3 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.30.0 + rev: v1.31.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue @@ -94,7 +94,7 @@ repos: # Terraform 0.13. # - id: terraform_validate - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v1.0.1 + rev: v2.0.0 hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier @@ -102,6 +102,6 @@ repos: hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.770 + rev: v0.782 hooks: - id: mypy From 907b89a20b17b5f8ffadbdc7ee71ed6cc31fd63b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 10 Jul 2020 21:34:22 -0400 Subject: [PATCH 159/253] Add the black profile to the isort configuration --- .isort.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.isort.cfg b/.isort.cfg index 5216e47..2fd6b08 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -10,3 +10,6 @@ import_heading_firstparty=cisagov Libraries known_third_party= # These must be manually set to correctly separate them from third party libraries known_first_party= + +# Run isort under the black profile to align with our other Python linting +profile=black From d6fe79dce37d8ecab907d7ba49fd580d4191819b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 02:19:02 -0400 Subject: [PATCH 160/253] Update actions/cache to v2 Switch the workflow to use cachev2. This puts us up to date and allows us to consolidate into one cache item. --- .github/workflows/build.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5fe2f8b..f6f4474 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,22 +23,18 @@ jobs: run: | echo "::set-env name=PY_VERSION::"\ "$(python -c "import platform;print(platform.python_version())")" - - name: Cache pip test requirements - uses: actions/cache@v1 + - name: Cache linting environments + uses: actions/cache@v2 with: - path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}-\ - ${{ hashFiles('**/requirements-test.txt') }}" - restore-keys: | - ${{ runner.os }}-pip-test-py${{ env.PY_VERSION }}- - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- - - name: Cache pre-commit hooks - uses: actions/cache@v1 - with: - path: ${{ env.PRE_COMMIT_CACHE_DIR }} - key: "${{ runner.os }}-pre-commit-py${{ env.PY_VERSION }}-\ + path: | + ${{ env.PIP_CACHE_DIR }} + ${{ env.PRE_COMMIT_CACHE_DIR }} + key: "${{ runner.os }}-lint-py${{ env.PY_VERSION }}-\ + ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" + restore-keys: | + ${{ runner.os }}-lint-py${{ env.PY_VERSION }}- + ${{ runner.os }}-lint- - name: Install dependencies run: | python -m pip install --upgrade pip From 6ab69f4f5158c3b2681f1b78a8fd71a57f0eb4d0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 02:21:04 -0400 Subject: [PATCH 161/253] Bump actions/setup-python to v2 Keep it up-to-date even though we are not currently using features added in v2. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f6f4474..2f58ed6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: 3.8 - name: Store installed Python version From 969efdd0bbc8cfad942ab0b731e24dcd43b8864f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 11:08:22 -0400 Subject: [PATCH 162/253] Adjust cache key name Make the job name the first part of the cache key since it is the only part that isn't variable. --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2f58ed6..36a908a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,12 +29,12 @@ jobs: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - key: "${{ runner.os }}-lint-py${{ env.PY_VERSION }}-\ + key: "lint-${{ runner.os }}-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | - ${{ runner.os }}-lint-py${{ env.PY_VERSION }}- - ${{ runner.os }}-lint- + lint-${{ runner.os }}-py${{ env.PY_VERSION }}- + lint-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip From 6edaffdded081d9149e22b957cfada94fd4831be Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 14:39:19 -0400 Subject: [PATCH 163/253] Update test job in build workflow to mirror updates to lint job --- .github/workflows/build.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57bafc0..4fef5ee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,18 +76,17 @@ jobs: needs: [build] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Cache pip test requirements - uses: actions/cache@v1 + - name: Cache testing environments + uses: actions/cache@v2 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-\ + key: "test-${{ runner.os }}-\ ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- + test-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip From 46593bf715e34730062b6343ee12efba63e35115 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 30 Jul 2020 14:40:48 -0400 Subject: [PATCH 164/253] Update release job in release workflow to use setup-python v2 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f851822..563ab08 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v2 with: python-version: 3.8 - name: Determine image version From 332e5a7669c39eff0feef36d054acb5766069fd7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:27:01 -0400 Subject: [PATCH 165/253] Run pre-commit autoupdate --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 52b6473..2689bb0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 + rev: v3.2.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -33,7 +33,7 @@ repos: args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.23.0 + rev: v1.24.2 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v2.7.0 + rev: v2.7.2 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -57,7 +57,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/python/black - rev: 19.10b0 + rev: 20.8b1 hooks: - id: black - repo: https://github.com/asottile/seed-isort-config @@ -65,16 +65,16 @@ repos: hooks: - id: seed-isort-config - repo: https://github.com/timothycrosley/isort - rev: 5.0.7 + rev: 5.5.0 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.3.0a3 + rev: v4.3.4 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.31.0 + rev: v1.37.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue @@ -98,7 +98,7 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.0.5 + rev: 2.1.1 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From 3e9441c837ce7d34a4043a8348f1bbcccaf2608b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:29:38 -0400 Subject: [PATCH 166/253] Update CODEOWNERS to resolve #50 --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a3619be..7735a52 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,4 +4,4 @@ # the repo. Unless a later match takes precedence, # these owners will be requested for review when someone # opens a pull request. -* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj @cisagov/team-ois +* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj From ae4aa123c81c0915bb1ce3da2ee19b41a98c9846 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:31:23 -0400 Subject: [PATCH 167/253] Update lint job to resolve #49 --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36a908a..7a92841 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,7 @@ jobs: ${{ env.PRE_COMMIT_CACHE_DIR }} key: "lint-${{ runner.os }}-py${{ env.PY_VERSION }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | lint-${{ runner.os }}-py${{ env.PY_VERSION }}- From d8f14e845b4054e8a45fc1f9a38c8537bcf30bfb Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Sep 2020 16:32:30 -0400 Subject: [PATCH 168/253] Sort .gitignore while I'm here --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e00826d..4b15d77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ +__pycache__ .mypy_cache .python-version -__pycache__ From 6a4164dfbac372fde6babd98ece2ab61c9b5e380 Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 7 Oct 2020 12:41:47 -0400 Subject: [PATCH 169/253] Bump build workflow python-version to 3.9 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7a92841..3493928 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Store installed Python version run: | echo "::set-env name=PY_VERSION::"\ From c9c00f1d03d2185dfa4a7cab0ec45bd76387c95a Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 7 Oct 2020 12:42:59 -0400 Subject: [PATCH 170/253] Replace deprecated set-env workflow syntax See: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/ --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3493928..d0563b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,8 +21,9 @@ jobs: python-version: 3.9 - name: Store installed Python version run: | - echo "::set-env name=PY_VERSION::"\ - "$(python -c "import platform;print(platform.python_version())")" + echo "PY_VERSION="\ + "$(python -c "import platform;print(platform.python_version())")" \ + >> $GITHUB_ENV - name: Cache linting environments uses: actions/cache@v2 with: From 7389a1b0cfa7e6d1b50fccef2381d3c28d31e49c Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 7 Oct 2020 12:43:54 -0400 Subject: [PATCH 171/253] Enable weekly dependabot scans of github-actions and pip --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1fd2ed3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +--- + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" From 04c046a6fae54f0584f82133ac31fa7f01a6d857 Mon Sep 17 00:00:00 2001 From: Felddy Date: Wed, 7 Oct 2020 14:37:56 -0400 Subject: [PATCH 172/253] Add Docker ecosystem to dependabot configuration --- .github/dependabot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1fd2ed3..fa93c02 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -2,6 +2,11 @@ version: 2 updates: + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "github-actions" directory: "/" schedule: From e64d87783d6fac5fd6eed881dfcbbf47556a82a0 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Oct 2020 13:26:11 -0400 Subject: [PATCH 173/253] Update build.yml Bump workflow test Python version to 3.9. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 071e31e..786781a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,7 +80,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Cache testing environments uses: actions/cache@v2 with: From 3de4b852b8635bf8948d4736156149e0ac701677 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2020 17:31:12 +0000 Subject: [PATCH 174/253] Bump actions/download-artifact from v1 to v2.0.5 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from v1 to v2.0.5. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v1...c3f5d00c8784369c43779f3d2611769594a61f7a) Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 786781a..c796b4c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -94,7 +94,7 @@ jobs: python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt - name: Download docker image artifact - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v2.0.5 with: name: dist - name: Load docker image From d28c382f48f7d48bd963c68626679a72cdbf7109 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2020 17:31:12 +0000 Subject: [PATCH 175/253] Bump actions/upload-artifact from v1 to v2.2.0 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from v1 to v2.2.0. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v1...27bce4eee761b5bc643f46a8dfb41b430c8d05f6) Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 786781a..a7d7387 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,7 +69,7 @@ jobs: version=$(./bump_version.sh show) docker save $IMAGE_NAME:latest | gzip > dist/image.tar.gz - name: Upload artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2.2.0 with: name: dist path: dist From df407a7fc08bcbf41963ae41cf94c6a2dcea4a00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2020 17:31:12 +0000 Subject: [PATCH 176/253] Bump python from 3.7-alpine to 3.9.0-alpine Bumps python from 3.7-alpine to 3.9.0-alpine. Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ba75023..67d0295 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG GIT_COMMIT=unspecified ARG GIT_REMOTE=unspecified ARG VERSION=unspecified -FROM python:3.7-alpine +FROM python:3.9.0-alpine ARG GIT_COMMIT ARG GIT_REMOTE From 1dcbc83d8c23e63d22fd25957374afb1b46b7497 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Oct 2020 13:42:01 -0400 Subject: [PATCH 177/253] Fix download-artifact incompatibility See: https://github.com/actions/download-artifact#compatibility-between-v1-and-v2 --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c796b4c..3a7d8e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,6 +97,7 @@ jobs: uses: actions/download-artifact@v2.0.5 with: name: dist + path: dist - name: Load docker image run: docker load < dist/image.tar.gz - name: Run tests From 53501f578ed38c7387746763c9efc648364b9a87 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Oct 2020 13:57:54 -0400 Subject: [PATCH 178/253] Relax upload-artifact version pin. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a7d7387..60bccc5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,7 +69,7 @@ jobs: version=$(./bump_version.sh show) docker save $IMAGE_NAME:latest | gzip > dist/image.tar.gz - name: Upload artifacts - uses: actions/upload-artifact@v2.2.0 + uses: actions/upload-artifact@v2 with: name: dist path: dist From 5307661b4c8bb163f32852b9420c6a333749e6d0 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 8 Oct 2020 14:03:56 -0400 Subject: [PATCH 179/253] Relax base image version pin. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 67d0295..b4f4e61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG GIT_COMMIT=unspecified ARG GIT_REMOTE=unspecified ARG VERSION=unspecified -FROM python:3.9.0-alpine +FROM python:3.9-alpine ARG GIT_COMMIT ARG GIT_REMOTE From 98f237f180b95060c2efe1cff1ab2e64521c048d Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 8 Oct 2020 14:33:04 -0400 Subject: [PATCH 180/253] Add missing cache keys for test job. Fixes #30 --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b52302a..f8267a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -86,7 +86,9 @@ jobs: with: path: ${{ env.PIP_CACHE_DIR }} key: "test-${{ runner.os }}-\ - ${{ hashFiles('**/requirements-test.txt') }}" + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | test-${{ runner.os }}- - name: Install dependencies From e14ecde029ecfbc39759055f7c03d7bc4ac7f458 Mon Sep 17 00:00:00 2001 From: Hillary Date: Fri, 9 Oct 2020 10:07:04 -0400 Subject: [PATCH 181/253] Add link to how to set up a new repo with skeletons --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 606df0a..66e6444 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,10 @@ appropriate for the major languages that we use. In many cases you will instead want to use one of the more specific skeleton projects derived from this one. +## New Repositories from a Skeleton ## + +Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup) for step-by-step instructions on how to start a new repository from a skeleton. This will save you time and effort when configuring a new repository! + ## Contributing ## We welcome contributions! Please see [here](CONTRIBUTING.md) for From 57d927d7d17f9b41aa49e5a57173005eb69719c9 Mon Sep 17 00:00:00 2001 From: Hillary Date: Fri, 9 Oct 2020 10:10:12 -0400 Subject: [PATCH 182/253] Add line breaks for linting Pre-commit is now installed properly. --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 66e6444..83da801 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,10 @@ skeleton projects derived from this one. ## New Repositories from a Skeleton ## -Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup) for step-by-step instructions on how to start a new repository from a skeleton. This will save you time and effort when configuring a new repository! +Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup) +for step-by-step instructions on how to start a new repository from +a skeleton. This will save you time and effort when configuring a +new repository! ## Contributing ## From e0beef276313fde355efc51fa2a56571763ff598 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Fri, 9 Oct 2020 14:22:05 -0400 Subject: [PATCH 183/253] Update .github/workflows/build.yml Co-authored-by: Nick M. <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f8267a2..4ff7cec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,8 +87,7 @@ jobs: path: ${{ env.PIP_CACHE_DIR }} key: "test-${{ runner.os }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" + ${{ hashFiles('**/requirements.txt') }}" restore-keys: | test-${{ runner.os }}- - name: Install dependencies From 0b979a796b02266599391c9a2ffe121af93b0754 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 9 Oct 2020 17:01:17 -0400 Subject: [PATCH 184/253] Use python 3.9 instead of 3.8 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 563ab08..01970b7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Determine image version id: get_ver run: | From 7d59f7e91d011bcbd5ea043999e6f7488078a7b9 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 12 Oct 2020 14:56:53 -0400 Subject: [PATCH 185/253] Fix several places where the old-style set-env was being used These changes all have to do with the version of the Docker image. --- .github/workflows/build.yml | 10 ++++------ .github/workflows/release.yml | 15 +++++++-------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4ff7cec..bf84602 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,22 +51,20 @@ jobs: steps: - uses: actions/checkout@v2 - name: Determine image version - id: get_ver - run: | - echo "##[set-output name=version;]$(./bump_version.sh show)" + run: echo IMAGE_VERSION=$(./bump_version.sh show) >> $GITHUB_ENV - name: Build docker image run: | - version=$(./bump_version.sh show) + version=${{ env.IMAGE_VERSION }} docker build \ --tag "$IMAGE_NAME" \ --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${{ steps.get_ver.outputs.version }} \ + --build-arg VERSION=${{ env.IMAGE_VERSION }} \ . - name: Save docker image artifact run: | mkdir dist - version=$(./bump_version.sh show) + version=${{ env.IMAGE_VERSION }} docker save $IMAGE_NAME:latest | gzip > dist/image.tar.gz - name: Upload artifacts uses: actions/upload-artifact@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 01970b7..dca4c36 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,34 +19,33 @@ jobs: with: python-version: 3.9 - name: Determine image version - id: get_ver - run: | - echo "##[set-output name=version;]$(./bump_version.sh show)" + run: echo IMAGE_VERSION=$(./bump_version.sh show) >> $GITHUB_ENV - name: Build Docker image run: | + version=${{ env.IMAGE_VERSION }} docker build \ --tag "$IMAGE_NAME" \ --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${{ steps.get_ver.outputs.version }} \ + --build-arg VERSION=${{ env.IMAGE_VERSION }} \ . - name: Tag Docker image run: | IFS='.' read -r -a version_array \ - <<< "${{ steps.get_ver.outputs.version }}" + <<< "${{ env.IMAGE_VERSION }}" docker login --username "$DOCKER_USER" --password "$DOCKER_PW" docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" docker tag "$IMAGE_NAME" \ - "${IMAGE_NAME}:${{ steps.get_ver.outputs.version }}" + "${IMAGE_NAME}:${{ env.IMAGE_VERSION }}" docker tag "$IMAGE_NAME" \ "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version_array[0]}" - name: Publish image to Docker Hub run: | IFS='.' read -r -a version_array \ - <<< "${{ steps.get_ver.outputs.version }}" + <<< "${{ env.IMAGE_VERSION }}" docker push "${IMAGE_NAME}:latest" - docker push "${IMAGE_NAME}:${{ steps.get_ver.outputs.version }}" + docker push "${IMAGE_NAME}:${{ env.IMAGE_VERSION }}" docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" docker push "${IMAGE_NAME}:${version_array[0]}" - name: Publish README.md to Docker Hub From b5ef97dd4bfb008e08c5abbc0e18848798b1c803 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 19 Oct 2020 11:49:41 -0400 Subject: [PATCH 186/253] Remove seed-isort-config pre-commit hook As of isort >=5, which we use, the seed-isort-config hook has been deprecated, and the repository archived. I have removed the hook and the configuration options it updated as a result. --- .isort.cfg | 5 ----- .pre-commit-config.yaml | 4 ---- 2 files changed, 9 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 2fd6b08..46d45f3 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -6,10 +6,5 @@ import_heading_stdlib=Standard Python Libraries import_heading_thirdparty=Third-Party Libraries import_heading_firstparty=cisagov Libraries -# Should be auto-populated by seed-isort-config hook -known_third_party= -# These must be manually set to correctly separate them from third party libraries -known_first_party= - # Run isort under the black profile to align with our other Python linting profile=black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2689bb0..84fbb49 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,10 +60,6 @@ repos: rev: 20.8b1 hooks: - id: black - - repo: https://github.com/asottile/seed-isort-config - rev: v2.2.0 - hooks: - - id: seed-isort-config - repo: https://github.com/timothycrosley/isort rev: 5.5.0 hooks: From de409db3f7567c342f6942ffef9a98dded20ede0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 19 Oct 2020 11:52:09 -0400 Subject: [PATCH 187/253] Use local filename in README link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 606df0a..b445f38 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ skeleton projects derived from this one. ## Contributing ## -We welcome contributions! Please see [here](CONTRIBUTING.md) for +We welcome contributions! Please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for details. ## License ## From 59eef4cc771b2340e131b216e4417b822e8d0b3f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 19 Oct 2020 11:53:09 -0400 Subject: [PATCH 188/253] Run pre-commit autoupdate --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 84fbb49..6ab5b5b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,13 +27,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.23.2 + rev: v0.24.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.24.2 + rev: v1.25.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -41,7 +41,7 @@ repos: hooks: - id: shell-lint - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 + rev: 3.8.4 hooks: - id: flake8 additional_dependencies: @@ -61,16 +61,16 @@ repos: hooks: - id: black - repo: https://github.com/timothycrosley/isort - rev: 5.5.0 + rev: 5.6.4 hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.3.4 + rev: v4.3.5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git - rev: v1.37.0 + rev: v1.43.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue @@ -94,10 +94,10 @@ repos: hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier - rev: 2.1.1 + rev: 2.1.2 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.782 + rev: v0.790 hooks: - id: mypy From 919460cb213ac2e3c507f1c2fb4c8e7cff54783b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 22 Oct 2020 13:42:10 -0400 Subject: [PATCH 189/253] Change repo for the prettier pre-commit hook Per https://github.com/prettier/prettier/issues/9459 and specifically https://github.com/prettier/prettier/issues/9459#issuecomment-713223710 the prettier hook for pre-commit is being broken out into its own repo. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6ab5b5b..7362f89 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,8 +93,8 @@ repos: rev: v2.0.0 hooks: - id: docker-compose-check - - repo: https://github.com/prettier/prettier - rev: 2.1.2 + - repo: https://github.com/prettier/pre-commit + rev: v2.1.2 hooks: - id: prettier - repo: https://github.com/pre-commit/mirrors-mypy From 4f14cc6213bfa811f85283124bced8ddb87b4feb Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 10:34:57 -0400 Subject: [PATCH 190/253] Add script to generate cross-platform Dockerfile. --- buildx-dockerfile.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 buildx-dockerfile.sh diff --git a/buildx-dockerfile.sh b/buildx-dockerfile.sh new file mode 100755 index 0000000..46710e9 --- /dev/null +++ b/buildx-dockerfile.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Create a Dockerfile suitable for a multi-platform build using buildx +# See: https://docs.docker.com/buildx/working-with-buildx/ + +set -o nounset +set -o errexit +set -o pipefail + +DOCKERFILE=Dockerfile +DOCKERFILEX=Dockerfile-x + +# We don't want this expression to expand. +# shellcheck disable=SC2016 +sed 's/^FROM /FROM --platform=$TARGETPLATFORM /g' < $DOCKERFILE > $DOCKERFILEX From ba039e872f453b14731863f6f6fb411ffcd5f33f Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 10:40:21 -0400 Subject: [PATCH 191/253] Add git ignore for generated Dockerfile-x. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9f81d3f..bceb4ee 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__ .mypy_cache .pytest_cache .python-version +Dockerfile-x From 3d0720bae1c0928402b11cf4174291c7929e8e03 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 10:44:59 -0400 Subject: [PATCH 192/253] Replace previous Docker workflows with a moderinzed, complete-workflow. These changes are based on the "complete workflow" from https://github.com/docker/build-push-action Additionally it adds support for the new "workflow_dispatch" event type. --- .github/workflows/build.yml | 229 ++++++++++++++++++++++++++++++---- .github/workflows/release.yml | 52 -------- push_readme.sh | 4 +- 3 files changed, 207 insertions(+), 78 deletions(-) delete mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bf84602..a489565 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,17 +3,37 @@ name: build on: push: + branches: + - '**' + tags: + - 'v*.*.*' pull_request: + schedule: + - cron: '0 10 * * *' # everyday at 10am repository_dispatch: types: [apb] + workflow_dispatch: + inputs: + remote-shell: + description: "Debug with remote shell" + required: true + default: false + image-tag: + description: "Tag to apply to pushed images" + required: true + default: dispatch env: + BUILDX_CACHE_DIR: ~/.cache/buildx IMAGE_NAME: cisagov/example PIP_CACHE_DIR: ~/.cache/pip + PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,\ + linux/arm64,linux/ppc64le,linux/s390x" PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit jobs: lint: + name: "Lint sources" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -46,32 +66,126 @@ jobs: run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files - build: + prepare: + name: "Prepare build variables" runs-on: ubuntu-latest + outputs: + created: ${{ steps.prep.outputs.created }} + repometa: ${{ steps.repo.outputs.result }} + source_version: ${{ steps.prep.outputs.source_version }} + tags: ${{ steps.prep.outputs.tags }} steps: - uses: actions/checkout@v2 - - name: Determine image version - run: echo IMAGE_VERSION=$(./bump_version.sh show) >> $GITHUB_ENV - - name: Build docker image - run: | - version=${{ env.IMAGE_VERSION }} - docker build \ - --tag "$IMAGE_NAME" \ - --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ - --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${{ env.IMAGE_VERSION }} \ - . - - name: Save docker image artifact + - name: Gather repository metadata + id: repo + uses: actions/github-script@v3 + with: + script: | + const repo = await github.repos.get(context.repo) + return repo.data + - name: Calculate output values + id: prep run: | - mkdir dist - version=${{ env.IMAGE_VERSION }} - docker save $IMAGE_NAME:latest | gzip > dist/image.tar.gz + VERSION=noop + SEMVER="^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?$" + if [ "${{ github.event_name }}" = "schedule" ]; then + VERSION=nightly + elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + VERSION=${{ github.event.inputs.image-tag }} + elif [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + elif [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') + if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; + then + VERSION=edge + fi + elif [[ $GITHUB_REF == refs/pull/* ]]; then + VERSION=pr-${{ github.event.number }} + fi + if [[ $VERSION =~ $SEMVER ]]; then + VERSION_NO_V=${VERSION#v} + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" + TAGS="${IMAGE_NAME}:${VERSION_NO_V//+/_},${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH},${IMAGE_NAME}:${MAJOR}.${MINOR},${IMAGE_NAME}:${MAJOR},${IMAGE_NAME}:latest" + else + TAGS="${IMAGE_NAME}:${VERSION}" + fi + if [ "${{ github.event_name }}" = "push" ]; then + TAGS="${TAGS},${IMAGE_NAME}:sha-${GITHUB_SHA::8}" + fi + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo ::set-output name=source_version::$(./bump_version.sh show) + echo ::set-output name=tags::${TAGS} + echo tags=${TAGS} + - name: Setup debug session remote shell + uses: mxschmitt/action-tmate@v3 + if: github.event.inputs.remote-shell == 'true' + build: + name: "Build test image" + runs-on: ubuntu-latest + needs: [prepare] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: ${{ env.BUILDX_CACHE_DIR }} + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Create dist directory + run: mkdir -p dist + - name: Build image + id: docker_build + uses: docker/build-push-action@v2 + with: + build-args: | + VERSION=${{ needs.prepare.outputs.source_version }} + cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} + cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} + context: . + file: ./Dockerfile + outputs: type=docker,dest=dist/image.tar + tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed + labels: "\ + org.opencontainers.image.created=${{ + needs.prepare.outputs.created }} + + org.opencontainers.image.description=${{ + fromJson(needs.prepare.outputs.repometa).description }} + + org.opencontainers.image.licenses=${{ + fromJson(needs.prepare.outputs.repometa).license.spdx_id }} + + org.opencontainers.image.revision=${{ github.sha }} + + org.opencontainers.image.source=${{ + fromJson(needs.prepare.outputs.repometa).clone_url }} + + org.opencontainers.image.title=${{ + fromJson(needs.prepare.outputs.repometa).name }} + + org.opencontainers.image.url=${{ + fromJson(needs.prepare.outputs.repometa).html_url }} + + org.opencontainers.image.version=${{ + needs.prepare.outputs.source_version }}" + - name: Compress image + run: gzip dist/image.tar - name: Upload artifacts uses: actions/upload-artifact@v2 with: name: dist path: dist test: + name: "Test image" runs-on: ubuntu-latest needs: [build] steps: @@ -79,21 +193,21 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Cache testing environments - uses: actions/cache@v2 + - name: Cache pip test requirements + uses: actions/cache@v1 with: path: ${{ env.PIP_CACHE_DIR }} - key: "test-${{ runner.os }}-\ - ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}" + key: "${{ runner.os }}-pip-test-\ + ${{ hashFiles('**/requirements-test.txt') }}" restore-keys: | - test-${{ runner.os }}- + ${{ runner.os }}-pip-test- + ${{ runner.os }}-pip- - name: Install dependencies run: | python -m pip install --upgrade pip pip install --upgrade --requirement requirements-test.txt - name: Download docker image artifact - uses: actions/download-artifact@v2.0.5 + uses: actions/download-artifact@v2 with: name: dist path: dist @@ -102,4 +216,71 @@ jobs: - name: Run tests env: RELEASE_TAG: ${{ github.event.release.tag_name }} - run: pytest + run: pytest --runslow + build-push-all: + name: "Build and push all platforms" + runs-on: ubuntu-latest + needs: [prepare, test] + if: github.event_name != 'pull_request' + steps: + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: ${{ env.BUILDX_CACHE_DIR }} + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Create cross-platform support Dockerfile-x + run: ./buildx-dockerfile.sh + - name: Build and push platform images to Docker Hub + id: docker_build + uses: docker/build-push-action@v2 + with: + build-args: | + VERSION=${{ needs.prepare.outputs.source_version }} + cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} + cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} + context: . + file: ./Dockerfile-x + platforms: ${{ env.PLATFORMS }} + push: true + tags: ${{ needs.prepare.outputs.tags }} + labels: "\ + org.opencontainers.image.created=${{ + needs.prepare.outputs.created }} + + org.opencontainers.image.description=${{ + fromJson(needs.prepare.outputs.repometa).description }} + + org.opencontainers.image.licenses=${{ + fromJson(needs.prepare.outputs.repometa).license.spdx_id }} + + org.opencontainers.image.revision=${{ github.sha }} + + org.opencontainers.image.source=${{ + fromJson(needs.prepare.outputs.repometa).clone_url }} + + org.opencontainers.image.title=${{ + fromJson(needs.prepare.outputs.repometa).name }} + + org.opencontainers.image.url=${{ + fromJson(needs.prepare.outputs.repometa).html_url }} + + org.opencontainers.image.version=${{ + needs.prepare.outputs.source_version }}" + - name: Publish README.md to Docker Hub + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + run: ./push_readme.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index dca4c36..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -name: release - -on: - release: - types: [prereleased, released] - -env: - IMAGE_NAME: cisagov/example - DOCKER_PW: ${{ secrets.DOCKER_PW }} - DOCKER_USER: ${{ secrets.DOCKER_USER }} - -jobs: - release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Determine image version - run: echo IMAGE_VERSION=$(./bump_version.sh show) >> $GITHUB_ENV - - name: Build Docker image - run: | - version=${{ env.IMAGE_VERSION }} - docker build \ - --tag "$IMAGE_NAME" \ - --build-arg GIT_COMMIT=$(git log -1 --format=%H) \ - --build-arg GIT_REMOTE=$(git remote get-url origin) \ - --build-arg VERSION=${{ env.IMAGE_VERSION }} \ - . - - name: Tag Docker image - run: | - IFS='.' read -r -a version_array \ - <<< "${{ env.IMAGE_VERSION }}" - docker login --username "$DOCKER_USER" --password "$DOCKER_PW" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:latest" - docker tag "$IMAGE_NAME" \ - "${IMAGE_NAME}:${{ env.IMAGE_VERSION }}" - docker tag "$IMAGE_NAME" \ - "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" - docker tag "$IMAGE_NAME" "${IMAGE_NAME}:${version_array[0]}" - - name: Publish image to Docker Hub - run: | - IFS='.' read -r -a version_array \ - <<< "${{ env.IMAGE_VERSION }}" - docker push "${IMAGE_NAME}:latest" - docker push "${IMAGE_NAME}:${{ env.IMAGE_VERSION }}" - docker push "${IMAGE_NAME}:${version_array[0]}.${version_array[1]}" - docker push "${IMAGE_NAME}:${version_array[0]}" - - name: Publish README.md to Docker Hub - run: ./push_readme.sh diff --git a/push_readme.sh b/push_readme.sh index 0b6d07a..4532701 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -3,7 +3,7 @@ # Push the README.md file to the docker hub repository # Requires the following environment variables to be set: -# DOCKER_PW, DOCKER_USER, IMAGE_NAME +# DOCKER_PASSWORD, DOCKER_USERNAME, IMAGE_NAME set -o nounset set -o errexit @@ -11,7 +11,7 @@ set -o pipefail token=$(curl -s -X POST \ -H "Content-Type: application/json" \ - -d '{"username": "'"$DOCKER_USER"'", "password": "'"$DOCKER_PW"'"}' \ + -d '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ https://hub.docker.com/v2/users/login/ | jq -r .token) code=$(jq -n --arg msg "$( Date: Fri, 30 Oct 2020 10:55:24 -0400 Subject: [PATCH 193/253] Add CodeQL workflow. Needed to check tests written in Python. --- .github/workflows/codeql-analysis.yml | 75 +++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..561e8c1 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,75 @@ +--- + +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + pull_request: + # The branches below must be a subset of the branches above + branches: [develop] + schedule: + - cron: '0 21 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', + # 'python'] + language: ['python'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a + # config file. By default, queries listed here will override any + # specified in a config file. Prefix the list here with "+" to use + # these queries and those in the config file. queries: + # ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or + # Java). If this step fails, then you should remove it and run the build + # manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language + + # - run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 7530392ec4ab87301360f91577f80dbc064da4bd Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 30 Oct 2020 14:54:59 -0400 Subject: [PATCH 194/253] Update README publisher to use new Docker URL. --- push_readme.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/push_readme.sh b/push_readme.sh index 4532701..2e55623 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -9,15 +9,17 @@ set -o nounset set -o errexit set -o pipefail +echo "Logging in and requesting JWT..." token=$(curl -s -X POST \ -H "Content-Type: application/json" \ -d '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ https://hub.docker.com/v2/users/login/ | jq -r .token) +echo "Pushing README file..." code=$(jq -n --arg msg "$( Date: Fri, 30 Oct 2020 15:06:29 -0400 Subject: [PATCH 195/253] Expand command option names to long form. --- push_readme.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/push_readme.sh b/push_readme.sh index 2e55623..13155cd 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -10,19 +10,20 @@ set -o errexit set -o pipefail echo "Logging in and requesting JWT..." -token=$(curl -s -X POST \ - -H "Content-Type: application/json" \ - -d '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ - https://hub.docker.com/v2/users/login/ | jq -r .token) +token=$(curl --silent --request POST \ + --header "Content-Type: application/json" \ + --data \ + '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ + https://hub.docker.com/v2/users/login/ | jq --raw-output .token) echo "Pushing README file..." -code=$(jq -n --arg msg "$( Date: Fri, 30 Oct 2020 18:44:23 -0400 Subject: [PATCH 196/253] Update Dockerfile to use opencontainers metadata and build-time labels. --- Dockerfile | 11 ++--------- tests/container_test.py | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index b4f4e61..c9184c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,11 @@ -ARG GIT_COMMIT=unspecified -ARG GIT_REMOTE=unspecified ARG VERSION=unspecified FROM python:3.9-alpine -ARG GIT_COMMIT -ARG GIT_REMOTE ARG VERSION -LABEL git_commit=${GIT_COMMIT} -LABEL git_remote=${GIT_REMOTE} -LABEL maintainer="mark.feldhousen@trio.dhs.gov" -LABEL vendor="Cyber and Infrastructure Security Agency" -LABEL version=${VERSION} +LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" +LABEL org.opencontainers.image.vendor="Cyber and Infrastructure Security Agency" ARG CISA_UID=421 ENV CISA_HOME="/home/cisa" diff --git a/tests/container_test.py b/tests/container_test.py index 90ea1a3..6153028 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -89,5 +89,5 @@ def test_container_version_label_matches(version_container): exec(f.read(), pkg_vars) # nosec project_version = pkg_vars["__version__"] assert ( - version_container.labels["version"] == project_version + version_container.labels["org.opencontainers.image.version"] == project_version ), "Dockerfile version label does not match project version" From e01481ba5b532010969242ff5d51d386c1ad56b7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 10:44:02 -0500 Subject: [PATCH 197/253] Prevent image pushes when linter checks fail. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a489565..7b79adb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -220,7 +220,7 @@ jobs: build-push-all: name: "Build and push all platforms" runs-on: ubuntu-latest - needs: [prepare, test] + needs: [lint, prepare, test] if: github.event_name != 'pull_request' steps: - name: Login to DockerHub From 297e52758195303ed6aa193f9b40acec7a56a98a Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 12:13:17 -0500 Subject: [PATCH 198/253] Modify cache keys to follow team-established naming conventions. Co-authored-by: Shane Frasier --- .github/workflows/build.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b79adb..7bb0a2f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -137,9 +137,9 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.BUILDX_CACHE_DIR }} - key: ${{ runner.os }}-buildx-${{ github.sha }} + key: buildx-${{ runner.os }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-buildx- + buildx-${{ runner.os }}- - name: Create dist directory run: mkdir -p dist - name: Build image @@ -194,14 +194,14 @@ jobs: with: python-version: 3.9 - name: Cache pip test requirements - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ${{ env.PIP_CACHE_DIR }} - key: "${{ runner.os }}-pip-test-\ - ${{ hashFiles('**/requirements-test.txt') }}" + key: "test-${{ runner.os }}-\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}" restore-keys: | - ${{ runner.os }}-pip-test- - ${{ runner.os }}-pip- + test-${{ runner.os }}- - name: Install dependencies run: | python -m pip install --upgrade pip @@ -238,9 +238,9 @@ jobs: uses: actions/cache@v2 with: path: ${{ env.BUILDX_CACHE_DIR }} - key: ${{ runner.os }}-buildx-${{ github.sha }} + key: buildx-${{ runner.os }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-buildx- + buildx-${{ runner.os }}- - name: Create cross-platform support Dockerfile-x run: ./buildx-dockerfile.sh - name: Build and push platform images to Docker Hub From f1e097a8641df648bd31673bfe911aae8540a05e Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 12:29:06 -0500 Subject: [PATCH 199/253] Add comments pointing to the opencontainers image-spec documentation. --- .github/workflows/build.yml | 2 ++ Dockerfile | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7bb0a2f..e80f5d4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -256,6 +256,8 @@ jobs: platforms: ${{ env.PLATFORMS }} push: true tags: ${{ needs.prepare.outputs.tags }} + # For a list of pre-defined annotation keys and value types see: + # https://github.com/opencontainers/image-spec/blob/master/annotations.md labels: "\ org.opencontainers.image.created=${{ needs.prepare.outputs.created }} diff --git a/Dockerfile b/Dockerfile index c9184c8..8819053 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,9 @@ FROM python:3.9-alpine ARG VERSION +# For a list of pre-defined annotation keys and value types see: +# https://github.com/opencontainers/image-spec/blob/master/annotations.md +# Note: Additional labels are added by the build workflow. LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" LABEL org.opencontainers.image.vendor="Cyber and Infrastructure Security Agency" From a88d550bd9ea10b572b58945981a76a804c00263 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 2 Nov 2020 16:36:43 -0500 Subject: [PATCH 200/253] Add comments to help document and break up a monolitic file. --- .github/workflows/build.yml | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e80f5d4..b5dd214 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,6 +11,7 @@ on: schedule: - cron: '0 10 * * *' # everyday at 10am repository_dispatch: + # Respond to rebuild requests. See: https://github.com/cisagov/action-apb/ types: [apb] workflow_dispatch: inputs: @@ -33,6 +34,8 @@ env: jobs: lint: + # Checks out the source and runs pre-commit hooks. Detects coding errors + # and style deviations. name: "Lint sources" runs-on: ubuntu-latest steps: @@ -66,7 +69,45 @@ jobs: run: pre-commit install-hooks - name: Run pre-commit on all files run: pre-commit run --all-files + prepare: + # Calculates and publishes outputs that are used by other jobs. + # + # Outputs: + # created: + # The current date-time in RFC3339 format. + # repometa: + # The json metadata describing this repository. + # source_version: + # The source version as reported by the `bump_version.sh show` command. + # tags: + # A comma separated list of Docker tags to be applied to the images on + # DockerHub. The tags will vary depending on: + # - The event that triggered the build. + # - The branch the build is based upon. + # - The git tag the build is based upon. + # + # When a build is based on a git tag of the form `v*.*.*` the image will + # be tagged on DockerHub with multiple levels of version specificity. + # For example, a git tag of `v1.2.3+a` will generate Docker tags of + # `:1.2.3_a`, `:1.2.3`, `:1.2`, `:1`, and `:latest`. + # + # Builds targeting the default branch will be tagged with `:edge`. + # + # Builds from other branches will be tagged with the branch name. + # Solidi `/` in branch names are replaced with hyphens `-` in the Docker + # tag. + # + # Builds triggered by a push event are tagged with a short hash in the + # form: sha-12345678 + # + # Builds triggered by a pull request are tagged with the pull request + # number in the form pr-123. + # + # Builds triggered using the GitHub GUI (workflow_dispatch) are tagged + # with the value specified by the user. + # + # Scheduled builds are tagged with `:nightly`. name: "Prepare build variables" runs-on: ubuntu-latest outputs: @@ -122,7 +163,10 @@ jobs: - name: Setup debug session remote shell uses: mxschmitt/action-tmate@v3 if: github.event.inputs.remote-shell == 'true' + build: + # Builds a single test image for the native platform. This image is saved + # as an artifact and loaded by the test job. name: "Build test image" runs-on: ubuntu-latest needs: [prepare] @@ -154,6 +198,8 @@ jobs: file: ./Dockerfile outputs: type=docker,dest=dist/image.tar tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed + # For a list of pre-defined annotation keys and value types see: + # https://github.com/opencontainers/image-spec/blob/master/annotations.md labels: "\ org.opencontainers.image.created=${{ needs.prepare.outputs.created }} @@ -184,7 +230,9 @@ jobs: with: name: dist path: dist + test: + # Executes tests on the single-platform image created in the "build" job. name: "Test image" runs-on: ubuntu-latest needs: [build] @@ -217,7 +265,13 @@ jobs: env: RELEASE_TAG: ${{ github.event.release.tag_name }} run: pytest --runslow + build-push-all: + # Builds the final set of images for each of the platforms listed in + # PLATFORMS environment variable. These images are tagged with the Docker + # tags calculated in the "prepare" job and pushed to DockerHub. The + # contents of README.md is pushed as the image's description. This job is + # skipped when the triggering event is a pull request. name: "Build and push all platforms" runs-on: ubuntu-latest needs: [lint, prepare, test] From a842abbfefefdb03bd1661dfb5956d7b09eeae25 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 3 Nov 2020 12:40:54 -0500 Subject: [PATCH 201/253] Insist that the cisagov devs are the owners of the .github directory This additional clause must remain at the _end_ of the CODEOWNERS file so that it cannot be overridden by a later clause. We want to make it so that all the .github files including CODEOWNERS are protected so only code owners (the dev team) can approve modifications to them. This will prevent configuration changes from breaking Actions and other management-type functions that the files in this directory control. By setting the .github files/folder to require code owner approval for changes, workflow and management changes will require dev team review and checking. Resolves #56. --- .github/CODEOWNERS | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7735a52..5671d70 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,10 @@ # Each line is a file pattern followed by one or more owners. -# These owners will be the default owners for everything in -# the repo. Unless a later match takes precedence, -# these owners will be requested for review when someone -# opens a pull request. +# These owners will be the default owners for everything in the +# repo. Unless a later match takes precedence, these owners will be +# requested for review when someone opens a pull request. * @dav3r @felddy @hillaryj @jsf9k @mcdonnnj + +# These folks own any files in the /.github directory at the root of +# the repository and any of its subdirectories. +/.github/ @dav3r @felddy @hillaryj @jsf9k @mcdonnnj From 5b199bc86974d09044df2e510d7ac4c6ba4ad850 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 3 Nov 2020 14:51:18 -0500 Subject: [PATCH 202/253] Remove offending slash Thanks to @dav3r and @mcdonnnj for the suggestion. --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5671d70..9c3d21f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,6 +5,6 @@ # requested for review when someone opens a pull request. * @dav3r @felddy @hillaryj @jsf9k @mcdonnnj -# These folks own any files in the /.github directory at the root of +# These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. /.github/ @dav3r @felddy @hillaryj @jsf9k @mcdonnnj From a29dc98d6f00c9d6ad6e05fc9f6b99163eb512db Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 5 Nov 2020 11:54:24 -0500 Subject: [PATCH 203/253] Apply correct typographic designation to character replacement comment. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b5dd214..63afd50 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -94,8 +94,8 @@ jobs: # # Builds targeting the default branch will be tagged with `:edge`. # - # Builds from other branches will be tagged with the branch name. - # Solidi `/` in branch names are replaced with hyphens `-` in the Docker + # Builds from other branches will be tagged with the branch name. Solidi + # `/` in branch names are replaced with hyphen-minuses `-` in the Docker # tag. # # Builds triggered by a push event are tagged with a short hash in the From e9426727634b5a8c198f83395262966290364f85 Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 5 Nov 2020 12:22:58 -0500 Subject: [PATCH 204/253] Modernize Docker shields in README to point to working links. --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index abfe21e..99246a9 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,9 @@ ## Docker Image ## -![MicroBadger Layers](https://img.shields.io/microbadger/layers/cisagov/example.svg) -![MicroBadger Size](https://img.shields.io/microbadger/image-size/cisagov/example.svg) +[![Docker Pulls](https://img.shields.io/docker/pulls/cisagov/example)](https://hub.docker.com/r/cisagov/example) +[![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/cisagov/example)](https://hub.docker.com/r/cisagov/example) +[![Platforms](https://img.shields.io/badge/platforms-amd64%20%7C%20arm%2Fv6%20%7C%20arm%2Fv7%20%7C%20arm64%20%7C%20ppc64le%20%7C%20s390x-blue)](https://hub.docker.com/r/cisagov/skeleton-docker/tags) This is a docker skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub docker project From da037f5045c34927038f264733abc51816e2b449 Mon Sep 17 00:00:00 2001 From: Mark Feldhousen Date: Thu, 5 Nov 2020 13:04:26 -0500 Subject: [PATCH 205/253] Elaborate on the Solidus. Add some additional comments in case anyone is confused about the nature of the solidus and its place within our tagging system. Co-authored-by: Shane Frasier --- .github/workflows/build.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 63afd50..d56e3eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -95,8 +95,11 @@ jobs: # Builds targeting the default branch will be tagged with `:edge`. # # Builds from other branches will be tagged with the branch name. Solidi - # `/` in branch names are replaced with hyphen-minuses `-` in the Docker - # tag. + # (`/` characters - commonly known as slashes) in branch names are + # replaced with hyphen-minuses (`-` characters) in the Docker tag. For + # more information about the solidus see these links: + # * https://www.compart.com/en/unicode/U+002F + # * https://en.wikipedia.org/wiki/Slash_(punctuation)#Encoding # # Builds triggered by a push event are tagged with a short hash in the # form: sha-12345678 From 02c5a6f8bf72e7695cc5957cd62a7e137f55f80e Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 10 Nov 2020 22:10:38 -0500 Subject: [PATCH 206/253] Use the python version output by actions/setup-python There is no need to run python code to determine the python version. Resolves #58. See here for details: https://github.com/actions/setup-python/blob/main/action.yml#L14-L16 --- .github/workflows/build.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d0563b8..bae558f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,26 +16,25 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - id: setup-python + uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Store installed Python version - run: | - echo "PY_VERSION="\ - "$(python -c "import platform;print(platform.python_version())")" \ - >> $GITHUB_ENV - name: Cache linting environments uses: actions/cache@v2 with: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - key: "lint-${{ runner.os }}-py${{ env.PY_VERSION }}-\ + key: | + lint-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | - lint-${{ runner.os }}-py${{ env.PY_VERSION }}- + lint-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}- lint-${{ runner.os }}- - name: Install dependencies run: | From 64c3fb61ab1eb832821d57882f42407e88970740 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 15 Nov 2020 23:02:59 -0500 Subject: [PATCH 207/253] Organize pre-commit hooks Group related pre-commit hooks together. Make sure that hooks are alphabetically sorted within those groups. --- .pre-commit-config.yaml | 48 +++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7362f89..e9dde69 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,30 +26,30 @@ repos: - --autofix - id: requirements-txt-fixer - id: trailing-whitespace + + # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.24.0 hooks: - id: markdownlint args: - --config=.mdl_config.json + - repo: https://github.com/prettier/pre-commit + rev: v2.1.2 + hooks: + - id: prettier - repo: https://github.com/adrienverge/yamllint rev: v1.25.0 hooks: - id: yamllint + + # Shell script hooks - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: - id: shell-lint - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 - hooks: - - id: flake8 - additional_dependencies: - - flake8-docstrings - - repo: https://github.com/asottile/pyupgrade - rev: v2.7.2 - hooks: - - id: pyupgrade + + # Python hooks - repo: https://github.com/PyCQA/bandit rev: 1.6.2 hooks: @@ -60,15 +60,33 @@ repos: rev: 20.8b1 hooks: - id: black + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 + hooks: + - id: flake8 + additional_dependencies: + - flake8-docstrings - repo: https://github.com/timothycrosley/isort rev: 5.6.4 hooks: - id: isort + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.790 + hooks: + - id: mypy + - repo: https://github.com/asottile/pyupgrade + rev: v2.7.2 + hooks: + - id: pyupgrade + + # Ansible hooks - repo: https://github.com/ansible/ansible-lint.git rev: v4.3.5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml + + # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.43.0 hooks: @@ -89,15 +107,9 @@ repos: # above have been resolved, which we hope will be with the release of # Terraform 0.13. # - id: terraform_validate + + # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit rev: v2.0.0 hooks: - id: docker-compose-check - - repo: https://github.com/prettier/pre-commit - rev: v2.1.2 - hooks: - - id: prettier - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.790 - hooks: - - id: mypy From 9b62ff0789df5d2d5c65f0674f13a12438dcb2b1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 18 Nov 2020 16:43:19 -0500 Subject: [PATCH 208/253] Fix test job missing Python version in cache key --- .github/workflows/build.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d56e3eb..5666fb8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -241,17 +241,21 @@ jobs: needs: [build] steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - id: setup-python + uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Cache pip test requirements + - name: Cache testing environments uses: actions/cache@v2 with: path: ${{ env.PIP_CACHE_DIR }} key: "test-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | + test-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}- test-${{ runner.os }}- - name: Install dependencies run: | From f4131e57d81557a751eec90ff0f07784809aa5d2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 20 Nov 2020 12:22:39 -0500 Subject: [PATCH 209/253] Clean up our actions/cache step Removed name because it was not more informative than the default. Swapped out a hardcoded job reference for the github.job context value. Switch the base cache key to a step environment value so we can set it once and reuse. Removed additional restore-key value that might have undesirable results. --- .github/workflows/build.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bae558f..5c65f71 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,22 +20,20 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - - name: Cache linting environments - uses: actions/cache@v2 + - uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-" with: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - key: | - lint-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-\ + key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | - lint-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}- - lint-${{ runner.os }}- + ${{ env.BASE_CACHE_KEY }} - name: Install dependencies run: | python -m pip install --upgrade pip From e3b0253bd16cd14efb69f732c028b5dbf5c4851c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 3 Dec 2020 11:26:53 -0500 Subject: [PATCH 210/253] Update remaining actions/cache uses Make sure all actions/cache steps are in-line with the changes made to the lint job's step. --- .github/workflows/build.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 05f5110..ce70bf5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -179,11 +179,13 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Cache Docker layers uses: actions/cache@v2 + env: + BASE_CACHE_KEY: buildx-${{ runner.os }}- with: path: ${{ env.BUILDX_CACHE_DIR }} - key: buildx-${{ runner.os }}-${{ github.sha }} + key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} restore-keys: | - buildx-${{ runner.os }}- + ${{ env.BASE_CACHE_KEY }} - name: Create dist directory run: mkdir -p dist - name: Build image @@ -244,16 +246,16 @@ jobs: python-version: 3.9 - name: Cache testing environments uses: actions/cache@v2 + env: + BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ + py${{ steps.setup-python.outputs.python-version }}-" with: path: ${{ env.PIP_CACHE_DIR }} - key: "test-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}-\ + key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | - test-${{ runner.os }}-\ - py${{ steps.setup-python.outputs.python-version }}- - test-${{ runner.os }}- + ${{ env.BASE_CACHE_KEY }} - name: Install dependencies run: | python -m pip install --upgrade pip @@ -294,11 +296,13 @@ jobs: uses: docker/setup-buildx-action@v1 - name: Cache Docker layers uses: actions/cache@v2 + env: + BASE_CACHE_KEY: buildx-${{ runner.os }}- with: path: ${{ env.BUILDX_CACHE_DIR }} - key: buildx-${{ runner.os }}-${{ github.sha }} + key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} restore-keys: | - buildx-${{ runner.os }}- + ${{ env.BASE_CACHE_KEY }} - name: Create cross-platform support Dockerfile-x run: ./buildx-dockerfile.sh - name: Build and push platform images to Docker Hub From b769825c41defa9365c108fbed03e79b753680c1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:03:13 -0500 Subject: [PATCH 211/253] Update prettier hook details Per https://github.com/prettier/prettier/pull/8937 the pre-commit hook has been moved to https://github.com/pre-commit/mirrors-prettier. I have also updated to the latest version in that repository. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9dde69..1d6845f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,8 +34,8 @@ repos: - id: markdownlint args: - --config=.mdl_config.json - - repo: https://github.com/prettier/pre-commit - rev: v2.1.2 + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.2.1 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint From 2d6bf50eab010ed699f5ed38f91fd4f397c473fd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:13:04 -0500 Subject: [PATCH 212/253] Update black hook repo url At some point the python GitHub organization renamed to psf (Python Software Foundation). Although it redirects with no issue, I think it should be updated to reflect the current repository URL. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d6845f..af393d0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,7 +56,7 @@ repos: - id: bandit args: - --config=.bandit.yml - - repo: https://github.com/python/black + - repo: https://github.com/psf/black rev: 20.8b1 hooks: - id: black From 5115baee52b77770f86f22af8618c432c8f0c064 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:15:03 -0500 Subject: [PATCH 213/253] Remove trailing '.git's from repository URLs Two hooks had trailing '.git's in the URLs. Although this is not a problem, we should be consistent in how we format things. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af393d0..fc1ed74 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,14 +80,14 @@ repos: - id: pyupgrade # Ansible hooks - - repo: https://github.com/ansible/ansible-lint.git + - repo: https://github.com/ansible/ansible-lint rev: v4.3.5 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - - repo: https://github.com/antonbabenko/pre-commit-terraform.git + - repo: https://github.com/antonbabenko/pre-commit-terraform rev: v1.43.0 hooks: - id: terraform_fmt From a7f86ddad0b4a70d0ee06ce8f6316749cb83e3f9 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:21:03 -0500 Subject: [PATCH 214/253] Enable two additional hooks from default pre-commit-hooks Enabled 'check-case-conflict' because of our mixed Linux and macOS development. Although APFS supports case-sensitive containers, it is not the default as far as I am aware. Linux filesystems are typically case-sensitive however. With the merge of https://github.com/cisagov/development-guide/pull/42 we now have a TOML file in a repository, so it does not hurt to add this hook in case more are added in the future. --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc1ed74..3e2c07c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,9 +7,11 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.2.0 hooks: + - id: check-case-conflict - id: check-executables-have-shebangs - id: check-json - id: check-merge-conflict + - id: check-toml - id: check-xml - id: debug-statements - id: detect-aws-credentials From 06159cbb65e0f5bf76ffe487b82684f0fc632f59 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Dec 2020 18:30:17 -0500 Subject: [PATCH 215/253] Run pre-commit autoupdate --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3e2c07c..cc668e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v3.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.24.0 + rev: v0.26.0 hooks: - id: markdownlint args: @@ -53,7 +53,7 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.6.2 + rev: 1.7.0 hooks: - id: bandit args: @@ -77,20 +77,20 @@ repos: hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.7.2 + rev: v2.7.4 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v4.3.5 + rev: v4.3.7 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.43.0 + rev: v1.45.0 hooks: - id: terraform_fmt # There are ongoing issues with how this command works. This issue From d836e91d323472fbdab0becb78bac7c79e3a2e73 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 17 Dec 2020 13:50:34 -0500 Subject: [PATCH 216/253] Update repo URL for the isort hook I missed that this repository was transfered from the creator, Timothy Crosley, to the PyCQA organization. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc668e1..e042de8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,7 +68,7 @@ repos: - id: flake8 additional_dependencies: - flake8-docstrings - - repo: https://github.com/timothycrosley/isort + - repo: https://github.com/PyCQA/isort rev: 5.6.4 hooks: - id: isort From 46ec3e7002c98c6653b39595008456eff1cd7f01 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 31 Dec 2020 15:06:18 -0500 Subject: [PATCH 217/253] Update ansible-lint hook URL --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e042de8..af4fed5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -82,7 +82,7 @@ repos: - id: pyupgrade # Ansible hooks - - repo: https://github.com/ansible/ansible-lint + - repo: https://github.com/ansible-community/ansible-lint rev: v4.3.7 hooks: - id: ansible-lint From 362b054459b15b6dd3ff5d7eb587682fee82f72a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 31 Dec 2020 15:08:07 -0500 Subject: [PATCH 218/253] Run pre-commit autoupdate --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af4fed5..2b3a71e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,7 +69,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.6.4 + rev: 5.7.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy From 8f7435a6fa7494c380b7b45831c7dec5ce3cd2bc Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 31 Dec 2020 15:09:53 -0500 Subject: [PATCH 219/253] Add the pre-commit-packer repo and hooks --- .pre-commit-config.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9dde69..28aa9bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -113,3 +113,10 @@ repos: rev: v2.0.0 hooks: - id: docker-compose-check + + # Packer hooks + - repo: https://github.com/cisagov/pre-commit-packer + rev: v0.0.2 + hooks: + - id: packer_validate + - id: packer_fmt From 2fb4e15cf6c743a3ce0ab434d3de7ae62ffc6fe2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 7 Jan 2021 04:07:38 -0500 Subject: [PATCH 220/253] Add the --strict flag to the yamllint pre-commot hook The --strict flag will cause the yamllint hook to exit with a non-zero exit code when warnings are found instead of only when errors are found. --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9dde69..26c06a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,6 +42,8 @@ repos: rev: v1.25.0 hooks: - id: yamllint + args: + - --strict # Shell script hooks - repo: https://github.com/detailyang/pre-commit-shell From ecdfc527d4338218f005684439214c2d126aa11b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 15 Jan 2021 10:34:34 -0500 Subject: [PATCH 221/253] Add the beautysh pre-commit hook This hook performs auto-formatting (beautifying) of Bash scripts. --- .pre-commit-config.yaml | 7 +++++++ setup-env | 14 +++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a3c7eb..42a2c7a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,6 +48,13 @@ repos: - --strict # Shell script hooks + - repo: https://github.com/lovesegfault/beautysh + rev: 6.0.1 + hooks: + - id: beautysh + args: + - --indent-size + - '2' - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: diff --git a/setup-env b/setup-env index 4d822c4..1579e04 100755 --- a/setup-env +++ b/setup-env @@ -50,14 +50,14 @@ while (( "$#" )); do shift ;; -*) # unsupported flags - echo "Error: Unsupported flag $1" >&2 - exit 1 - ;; + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; *) # preserve positional arguments - PARAMS="$PARAMS $1" - shift - ;; - esac + PARAMS="$PARAMS $1" + shift + ;; + esac done # set positional arguments in their proper place From 6bd93ff6fbd1e9fe1a6e508d6a65225b63220a53 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sun, 31 Jan 2021 19:24:54 -0500 Subject: [PATCH 222/253] Update pre-commit hooks with `pre-commit autoupdate` --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a3c7eb..e1da8e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.25.0 + rev: v1.26.0 hooks: - id: yamllint args: @@ -75,17 +75,17 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.790 + rev: v0.800 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v2.7.4 + rev: v2.10.0 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v4.3.7 + rev: v5.0.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From 6e33a4e5fb7e4e91e14971e33283902f11e3d9c1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 16 Feb 2021 10:44:50 -0500 Subject: [PATCH 223/253] Add a requirements-dev.txt pip requirements file Our other skeletons have this file, even if they are not Python projects. The directions in CONTRIBUTING.md reference such a file, which can be misleading. Rather than amending the CONTRIBUTING documentation, I feel it's better to align with our other skeletons for consistency. --- requirements-dev.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..d84ee68 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +--requirement requirements-test.txt +ipython From c97a883ada7ae0cbd50f790d4deb4e79639736dd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 16 Feb 2021 13:18:08 -0500 Subject: [PATCH 224/253] Revert ansible-lint version update The v5 release of ansible-lint introduces breaking changes, so we are holding off on updating until things are resolved. Conversation about this can be tracked in https://github.com/cisagov/skeleton-ansible-role/issues/69. --- .pre-commit-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1da8e2..4fdfdff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -85,7 +85,9 @@ repos: # Ansible hooks - repo: https://github.com/ansible-community/ansible-lint - rev: v5.0.0 + # This is intentionally being held back because of issues in v5 per + # https://github.com/cisagov/skeleton-ansible-role/issues/69 + rev: v4.3.7 hooks: - id: ansible-lint # files: molecule/default/playbook.yml From ab1bcd5ddea599ad614da235f6b43e72630d8e77 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 16 Feb 2021 17:47:35 -0500 Subject: [PATCH 225/253] Autoformat bump_version.sh with beautysh --- bump_version.sh | 58 ++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/bump_version.sh b/bump_version.sh index 7c23343..81a394f 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -14,34 +14,34 @@ old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) if [ $# -ne 1 ] then - echo "$HELP_INFORMATION" + echo "$HELP_INFORMATION" else - case $1 in - major|minor|patch|prerelease|build) - new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - git add $VERSION_FILE - git commit -m"Bumping version from $old_version to $new_version" - git push - ;; - finalize) - new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - git add $VERSION_FILE - git commit -m"Bumping version from $old_version to $new_version" - git push - ;; - show) - echo "$old_version" - ;; - *) - echo "$HELP_INFORMATION" - ;; - esac + case $1 in + major|minor|patch|prerelease|build) + new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bumping version from $old_version to $new_version" + git push + ;; + finalize) + new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") + echo Changing version from "$old_version" to "$new_version" + tmp_file=/tmp/version.$$ + sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file + mv $tmp_file $VERSION_FILE + git add $VERSION_FILE + git commit -m"Bumping version from $old_version to $new_version" + git push + ;; + show) + echo "$old_version" + ;; + *) + echo "$HELP_INFORMATION" + ;; + esac fi From ae83ed86e32b3a70e98881efae218e400d789267 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 16 Feb 2021 17:48:28 -0500 Subject: [PATCH 226/253] Autoformat push_readme.sh with beautysh --- push_readme.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/push_readme.sh b/push_readme.sh index 13155cd..12e2340 100755 --- a/push_readme.sh +++ b/push_readme.sh @@ -11,19 +11,19 @@ set -o pipefail echo "Logging in and requesting JWT..." token=$(curl --silent --request POST \ - --header "Content-Type: application/json" \ - --data \ - '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ + --header "Content-Type: application/json" \ + --data \ + '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ https://hub.docker.com/v2/users/login/ | jq --raw-output .token) echo "Pushing README file..." code=$(jq --null-input --arg msg "$( Date: Tue, 16 Feb 2021 22:39:33 -0500 Subject: [PATCH 227/253] Fix the agency name in the vendor label --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8819053..1c48a45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ARG VERSION # https://github.com/opencontainers/image-spec/blob/master/annotations.md # Note: Additional labels are added by the build workflow. LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" -LABEL org.opencontainers.image.vendor="Cyber and Infrastructure Security Agency" +LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" ARG CISA_UID=421 ENV CISA_HOME="/home/cisa" From 9f3efb52537f19a41211c9346dfd03e898fee8fd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 26 Feb 2021 16:43:38 -0500 Subject: [PATCH 228/253] Update CodeQL workflow for GitHub Actions I noticed the following warning when looking at GHA workflow runs: 1 issue was detected with this workflow: git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results. After looking into it I updated the workflow per this page on GitHub Docs: https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/troubleshooting-the-codeql-workflow#warning-git-checkout-head2-is-no-longer-necessary --- .github/workflows/codeql-analysis.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 561e8c1..127ef08 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -33,15 +33,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 - - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL From 78b090bb5060ba81d99c16f75c575735935476c3 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 14:09:29 -0400 Subject: [PATCH 229/253] Remove duplicate LICENSE.md file --- LICENSE.md | 116 ----------------------------------------------------- 1 file changed, 116 deletions(-) delete mode 100644 LICENSE.md diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index 670154e..0000000 --- a/LICENSE.md +++ /dev/null @@ -1,116 +0,0 @@ -CC0 1.0 Universal - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator and -subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for the -purpose of contributing to a commons of creative, cultural and scientific -works ("Commons") that the public can reliably and without fear of later -claims of infringement build upon, modify, incorporate in other works, reuse -and redistribute as freely as possible in any form whatsoever and for any -purposes, including without limitation commercial purposes. These owners may -contribute to the Commons to promote the ideal of a free culture and the -further production of creative, cultural and scientific works, or to gain -reputation or greater distribution for their Work in part through the use and -efforts of others. - -For these and/or other purposes and motivations, and without any expectation -of additional consideration or compensation, the person associating CC0 with a -Work (the "Affirmer"), to the extent that he or she is an owner of Copyright -and Related Rights in the Work, voluntarily elects to apply CC0 to the Work -and publicly distribute the Work under its terms, with knowledge of his or her -Copyright and Related Rights in the Work and the meaning and intended legal -effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not limited -to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, communicate, - and translate a Work; - - ii. moral rights retained by the original author(s) and/or performer(s); - - iii. publicity and privacy rights pertaining to a person's image or likeness - depicted in a Work; - - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - - v. rights protecting the extraction, dissemination, use and reuse of data in - a Work; - - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation thereof, - including any amended or successor version of such directive); and - - vii. other similar, equivalent or corresponding rights throughout the world - based on applicable law or treaty, and any national implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention of, -applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and -unconditionally waives, abandons, and surrenders all of Affirmer's Copyright -and Related Rights and associated claims and causes of action, whether now -known or unknown (including existing as well as future claims and causes of -action), in the Work (i) in all territories worldwide, (ii) for the maximum -duration provided by applicable law or treaty (including future time -extensions), (iii) in any current or future medium and for any number of -copies, and (iv) for any purpose whatsoever, including without limitation -commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes -the Waiver for the benefit of each member of the public at large and to the -detriment of Affirmer's heirs and successors, fully intending that such Waiver -shall not be subject to revocation, rescission, cancellation, termination, or -any other legal or equitable action to disrupt the quiet enjoyment of the Work -by the public as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason be -judged legally invalid or ineffective under applicable law, then the Waiver -shall be preserved to the maximum extent permitted taking into account -Affirmer's express Statement of Purpose. In addition, to the extent the Waiver -is so judged Affirmer hereby grants to each affected person a royalty-free, -non transferable, non sublicensable, non exclusive, irrevocable and -unconditional license to exercise Affirmer's Copyright and Related Rights in -the Work (i) in all territories worldwide, (ii) for the maximum duration -provided by applicable law or treaty (including future time extensions), (iii) -in any current or future medium and for any number of copies, and (iv) for any -purpose whatsoever, including without limitation commercial, advertising or -promotional purposes (the "License"). The License shall be deemed effective as -of the date CC0 was applied by Affirmer to the Work. Should any part of the -License for any reason be judged legally invalid or ineffective under -applicable law, such partial invalidity or ineffectiveness shall not -invalidate the remainder of the License, and in such case Affirmer hereby -affirms that he or she will not (i) exercise any of his or her remaining -Copyright and Related Rights in the Work or (ii) assert any associated claims -and causes of action with respect to the Work, in either case contrary to -Affirmer's express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - - b. Affirmer offers the Work as-is and makes no representations or warranties - of any kind concerning the Work, express, implied, statutory or otherwise, - including without limitation warranties of title, merchantability, fitness - for a particular purpose, non infringement, or the absence of latent or - other defects, accuracy, or the present or absence of errors, whether or not - discoverable, all to the greatest extent permissible under applicable law. - - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without limitation - any person's Copyright and Related Rights in the Work. Further, Affirmer - disclaims responsibility for obtaining any necessary consents, permissions - or other rights required for any use of the Work. - - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to this - CC0 or use of the Work. - -For more information, please see - From 0e9f3101533a52f67f561976b10ce0742d22fd83 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 14:13:28 -0400 Subject: [PATCH 230/253] Use the correct version.txt file --- src/version.txt | 2 +- version.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 version.txt diff --git a/src/version.txt b/src/version.txt index f102a9c..3a3cd8c 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.0.1" +1.3.1 diff --git a/version.txt b/version.txt deleted file mode 100644 index 3a3cd8c..0000000 --- a/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.3.1 From 89658c12499078a0685984852e42f1fb774a55e1 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 14:14:59 -0400 Subject: [PATCH 231/253] Remove now-unnecessary TravisCI files --- .travis.yml | 19 ------------------- travis_scripts/build_docker_image.sh | 10 ---------- travis_scripts/deploy_to_docker_hub.sh | 11 ----------- 3 files changed, 40 deletions(-) delete mode 100644 .travis.yml delete mode 100644 travis_scripts/build_docker_image.sh delete mode 100644 travis_scripts/deploy_to_docker_hub.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 488b301..0000000 --- a/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -sudo: required - -services: - - docker - -env: - global: - - IMAGE_NAME=cisagov/gatherer - - DOCKER_USER=jsf9k - - secure: "zIyHkDg9thZRegsFjQDTSqpuqkybIQSrQarUfsXJLJR1rAm7s1qeDT1I3AlNyY8x4RgY5PreFqRM3ebsFpHSUmXFS0ECiYVO6aJQ409Hx5wUhbEN1hpwocVJ8iC4tEf7/xv5Lu8LTUlD5/wwEhkcgs5p5OTFfEXu9iZGPQT7lZQMYlch855DvzGNr0TON/biPQ3QK70QtcyJyLsIErQbSkPw7SvhuPrY/HOW/CqbgkVkCqQQL9/M8FfwUzV/iIftAWFU2+vEN2leNmqI69CLEToyhljVK80uMoNJtC3NCeVVnaLhLtLObdASPt1+MIngKe/wxhciNXALAxLr87+MeeouYj/VrF34DZa3+qRcwby7nfTZIBFqpJ4ne+Nf63XXtbhNIkEL43kPILCu2nx1EHvlOCVYOJV4dKYxpAmF+/DERxJDQH/ZL0ltAb5j8nac1HdB/AnrKEhMvSZiOUT1lx1y2x4rQeThOYb9+Qaxejukxcuq0ykreJa7hxSiZ5o4hXlX24PAq3awqDSHk5GHBA6WaBQifuOoYIqypOm4JfvzfzXHYSmWSCZCG6NBKSX8iTIrHhQ8Anhc60zZwXudfhKzeUxa8HTh4jHtGI2dJxON1ajoxWnUovKuNPtfvl+yIjAN9cU4KazFnMu0jnJMxGubhWmAieHM5aeF4MlUQRw=" - -script: - - bash travis_scripts/build_docker_image.sh - -deploy: - - provider: script - script: bash travis_scripts/deploy_to_docker_hub.sh - on: - tags: true diff --git a/travis_scripts/build_docker_image.sh b/travis_scripts/build_docker_image.sh deleted file mode 100644 index 9b8b3b5..0000000 --- a/travis_scripts/build_docker_image.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o errexit -set -o pipefail - -# semver uses a plus character for the build number (if present). -# This is invalid for a Docker tag, so we replace it with a minus. -version=$(./bump_version.sh show|sed "s/+/-/") -docker build -t "$IMAGE_NAME":"$version" . diff --git a/travis_scripts/deploy_to_docker_hub.sh b/travis_scripts/deploy_to_docker_hub.sh deleted file mode 100644 index cb04d01..0000000 --- a/travis_scripts/deploy_to_docker_hub.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o errexit -set -o pipefail - -echo "$DOCKER_PW" | docker login -u "$DOCKER_USER" --password-stdin -# semver uses a plus character for the build number (if present). -# This is invalid for a Docker tag, so we replace it with a minus. -version=$(./bump_version.sh show|sed "s/+/-/") -docker push "$IMAGE_NAME":"$version" From a57caafa1375c6b71d0d851f050043eaa1a8f870 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 14:21:03 -0400 Subject: [PATCH 232/253] Make Dockerfile non-executable I'm not sure why it was executable to begin with. --- Dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile old mode 100755 new mode 100644 From 6beaec2ec3a3d26a355d255c1047c3264360df5c Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 14:21:44 -0400 Subject: [PATCH 233/253] Quote a shell variable to make beautysh happy --- tag.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tag.sh b/tag.sh index f72316d..fbb93cc 100755 --- a/tag.sh +++ b/tag.sh @@ -2,4 +2,4 @@ version=$(./bump_version.sh show) -git tag v$version && git push --tags +git tag v"$version" && git push --tags From c5afd2085368038f13b3f7c028b8161009733258 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 14:23:56 -0400 Subject: [PATCH 234/253] Remove extraneous blank line Also remove the emojis from the title line since they are triggering the pre-commit Markdown linter. --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 6418cdd..bc5256f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# gatherer :notebook: :file_folder: # +# gatherer # [![GitHub Build Status](https://github.com/cisagov/gatherer/workflows/build/badge.svg)](https://github.com/cisagov/gatherer/actions) [![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/gatherer.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/gatherer/alerts/) @@ -57,7 +57,6 @@ This container supports no environment variables. |-------------|----------------| | /home/gatherer/shared | Output | - ## Contributing ## We welcome contributions! Please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for From 7ce8dbb061c3157b11da0843c42fc750a4ae54b1 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 14:30:08 -0400 Subject: [PATCH 235/253] Make the black and flake8 linters happy --- scripts/fed_hostnames.py | 118 +++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 60 deletions(-) diff --git a/scripts/fed_hostnames.py b/scripts/fed_hostnames.py index 2e31836..1cc0055 100755 --- a/scripts/fed_hostnames.py +++ b/scripts/fed_hostnames.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -"""Output a list of all detected Federal hostnames +"""Output a list of all detected Federal hostnames. Usage: COMMAND_NAME [--db-creds-file=FILENAME] [--debug] [--output-file=FILENAME] @@ -20,27 +20,25 @@ """ +# Standard Python Libraries import logging import re +# Third-Party Libraries from docopt import docopt from mongo_db_from_config import db_from_config import pymongo.errors import yaml - # The ports that are most commonly used by public-facing web servers -WebServerPorts = { - 80, 280, 443, 591, 593, 832, 8080, 8888, 4443, 8443, 9443, 10443 -} +WebServerPorts = {80, 280, 443, 591, 593, 832, 8080, 8888, 4443, 8443, 9443, 10443} # The ports that are most commonly used by mail servers MailServerPorts = {25, 110, 143, 465, 587, 993, 995, 2525} def get_all_descendants(database, owner): - """Return all (non-retired) descendents of the given Cyber Hygiene - parent + """Return all (non-retired) descendents of the Cyber Hygiene parent. Parameters ---------- @@ -55,14 +53,14 @@ def get_all_descendants(database, owner): ------- list of str: The descendents of the Cyber Hygiene parent. """ - current_request = database.requests.find_one({'_id': owner}) + current_request = database.requests.find_one({"_id": owner}) if not current_request: - raise ValueError(owner + ' has no request document') + raise ValueError(owner + " has no request document") descendants = [] - if current_request.get('children'): - for child in current_request['children']: - if not database.requests.find_one({'_id': child}).get('retired'): + if current_request.get("children"): + for child in current_request["children"]: + if not database.requests.find_one({"_id": child}).get("retired"): descendants.append(child) descendants += get_all_descendants(database, child) @@ -70,90 +68,90 @@ def get_all_descendants(database, owner): def main(): + """Output a list of all detected Federal hostnames.""" global __doc__ - __doc__ = re.sub('COMMAND_NAME', __file__, __doc__) - args = docopt(__doc__, version='v0.0.1') + __doc__ = re.sub("COMMAND_NAME", __file__, __doc__) + args = docopt(__doc__, version="v0.0.1") # Set up logging log_level = logging.WARNING - if args['--debug']: + if args["--debug"]: log_level = logging.DEBUG - logging.basicConfig(format='%(asctime)-15s %(levelname)s %(message)s', - level=log_level) + logging.basicConfig( + format="%(asctime)-15s %(levelname)s %(message)s", level=log_level + ) - db_creds_file = args['--db-creds-file'] + db_creds_file = args["--db-creds-file"] try: db = db_from_config(db_creds_file) except OSError: - logging.critical('Database configuration file {} does not exist'.format(db_creds_file), - exc_info=True) + logging.critical( + "Database configuration file {} does not exist".format(db_creds_file), + exc_info=True, + ) return 1 except yaml.YAMLError: - logging.critical('Database configuration file {} does not contain valid YAML'.format(db_creds_file), - exc_info=True) + logging.critical( + "Database configuration file {} does not contain valid YAML".format( + db_creds_file + ), + exc_info=True, + ) return 1 except KeyError: - logging.critical('Database configuration file {} does not contain the expected keys'.format(db_creds_file), - exc_info=True) + logging.critical( + "Database configuration file {} does not contain the expected keys".format( + db_creds_file + ), + exc_info=True, + ) return 1 except pymongo.errors.ConnectionError: - logging.critical('Unable to connect to the database server in {}'.format(db_creds_file), - exc_info=True) + logging.critical( + "Unable to connect to the database server in {}".format(db_creds_file), + exc_info=True, + ) return 1 except pymongo.errors.InvalidName: - logging.critical('The database in {} does not exist'.format(db_creds_file), - exc_info=True) + logging.critical( + "The database in {} does not exist".format(db_creds_file), exc_info=True + ) return 1 # Get all Federal organizations - fed_orgs = get_all_descendants(db, 'FEDERAL') - logging.debug('Federal orgs are {}'.format(fed_orgs)) + fed_orgs = get_all_descendants(db, "FEDERAL") + logging.debug("Federal orgs are {}".format(fed_orgs)) # Get all Federal hosts with open ports that indicate a possible web or # email server (latest scan only)... potential_web_or_email_server_ips = { - i['ip_int'] for i in db.port_scans.find( + i["ip_int"] + for i in db.port_scans.find( { - 'latest': True, - 'owner': { - '$in': fed_orgs - }, - 'port': { - '$in': list(WebServerPorts | MailServerPorts) - } + "latest": True, + "owner": {"$in": fed_orgs}, + "port": {"$in": list(WebServerPorts | MailServerPorts)}, }, - { - '_id': False, - 'ip_int': True - } + {"_id": False, "ip_int": True}, ) } # ...of these, get all Federal hosts with a detected hostname (latest scan # only) fed_hosts = db.host_scans.find( { - 'latest': True, - 'ip_int': { - '$in': list(potential_web_or_email_server_ips) - }, - 'owner': { - '$in': fed_orgs - }, - 'hostname': { - '$ne': None - } + "latest": True, + "ip_int": {"$in": list(potential_web_or_email_server_ips)}, + "owner": {"$in": fed_orgs}, + "hostname": {"$ne": None}, }, - { - '_id': False, - 'hostname': True, - 'owner': True} + {"_id": False, "hostname": True, "owner": True}, ) - with open(args['--output-file'], 'w') as file: + with open(args["--output-file"], "w") as file: for host in fed_hosts: - file.write('{},{}\n'.format(host['hostname'], host['owner'])) - logging.debug('Federal host {}'.format(host)) + file.write("{},{}\n".format(host["hostname"], host["owner"])) + logging.debug("Federal host {}".format(host)) -if __name__ == '__main__': +if __name__ == "__main__": main() From 5f323586eb675a607eea263b2e6c3a16bb693940 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 14:30:47 -0400 Subject: [PATCH 236/253] Make beautysh happy --- gather-domains.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/gather-domains.sh b/gather-domains.sh index e9e6108..46c77a5 100755 --- a/gather-domains.sh +++ b/gather-domains.sh @@ -9,7 +9,7 @@ OUTPUT_DIR=$HOME_DIR/shared/artifacts # Create the output directory, if necessary if [ ! -d $OUTPUT_DIR ] then - mkdir $OUTPUT_DIR + mkdir $OUTPUT_DIR fi ### @@ -25,14 +25,14 @@ scripts/fed_hostnames.py --output-file=$OUTPUT_DIR/cyhy_fed_hostnames.csv # here. ### wget https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-federal.csv \ - -O $OUTPUT_DIR/current-federal.csv + -O $OUTPUT_DIR/current-federal.csv ### # Grab our online list of extra, non-.gov domains that the # corresponding stakeholder has requested we scan. We have verified # that the stakeholder controls these domains. ### wget https://raw.githubusercontent.com/cisagov/scan-target-data/develop/current-federal-non-dotgov.csv \ - -O $OUTPUT_DIR/current-federal-non-dotgov.csv + -O $OUTPUT_DIR/current-federal-non-dotgov.csv ### # Concatenate current-federal.csv with the list of extra, non-.gov # domains. @@ -41,10 +41,10 @@ wget https://raw.githubusercontent.com/cisagov/scan-target-data/develop/current- # before the concatenation. ### tail -n +2 $OUTPUT_DIR/current-federal-non-dotgov.csv > \ - /tmp/current-federal-non-dotgov.csv + /tmp/current-federal-non-dotgov.csv cat $OUTPUT_DIR/current-federal.csv \ - /tmp/current-federal-non-dotgov.csv > \ - $OUTPUT_DIR/current-federal_modified.csv + /tmp/current-federal-non-dotgov.csv > \ + $OUTPUT_DIR/current-federal_modified.csv ### # Remove the FED.US domain. This is really a top-level domain, # analogous to .gov or .com. It is only present in current-federal as @@ -58,7 +58,7 @@ sed -i '/^FED\.US,.*/d' $OUTPUT_DIR/current-federal_modified.csv # Also remove all other domains that belong to the judicial branch. ### sed -i '/[^,]*,[^,]*,U\.S\. Courts,/d;/[^,]*,[^,]*,The Supreme Court,/d' \ - $OUTPUT_DIR/current-federal_modified.csv + $OUTPUT_DIR/current-federal_modified.csv ### # Remove all domains that belong to the legislative branch, with the # exception of the House of Representatives (HOR). HOR specifically @@ -70,7 +70,7 @@ sed -i '/[^,]*,[^,]*,U\.S\. Courts,/d;/[^,]*,[^,]*,The Supreme Court,/d' \ # (Congress)" in current-federal. ### sed -i '/[^,]*,[^,]*,Library of Congress,/d;/[^,]*,[^,]*,Government Publishing Office,/d;/[^,]*,[^,]*,Congressional Office of Compliance,/d;/[^,]*,[^,]*,Stennis Center for Public Service,/d;/[^,]*,[^,]*,U.S. Capitol Police,/d;/[^,]*,[^,]*,Architect of the Capitol,/d' \ - $OUTPUT_DIR/current-federal_modified.csv + $OUTPUT_DIR/current-federal_modified.csv ### # Gather hostnames using GSA/data, analytics.usa.gov, Censys, EOT, @@ -87,16 +87,16 @@ sed -i '/[^,]*,[^,]*,Library of Congress,/d;/[^,]*,[^,]*,Government Publishing O # include/current-federal-non-dotgov.csv ### $HOME_DIR/domain-scan/gather current_federal,analytics_usa_gov,censys_snapshot,rapid,eot_2012,eot_2016,cyhy,other \ - --suffix=.gov,.edu,.com,.net,.org,.us --ignore-www --include-parents \ - --parents=$OUTPUT_DIR/current-federal_modified.csv \ - --current_federal=$OUTPUT_DIR/current-federal_modified.csv \ - --analytics_usa_gov=https://analytics.usa.gov/data/live/sites.csv \ - --censys_snapshot=https://raw.githubusercontent.com/GSA/data/master/dotgov-websites/censys-federal-snapshot.csv \ - --rapid=https://raw.githubusercontent.com/GSA/data/master/dotgov-websites/rdns-federal-snapshot.csv \ - --eot_2012=https://raw.githubusercontent.com/cisagov/scan-target-data/develop/eot-2012.csv \ - --eot_2016=https://raw.githubusercontent.com/cisagov/scan-target-data/develop/eot-2016.csv \ - --cyhy=$OUTPUT_DIR/cyhy_fed_hostnames.csv \ - --other=https://raw.githubusercontent.com/GSA/data/master/dotgov-websites/other-websites.csv + --suffix=.gov,.edu,.com,.net,.org,.us --ignore-www --include-parents \ + --parents=$OUTPUT_DIR/current-federal_modified.csv \ + --current_federal=$OUTPUT_DIR/current-federal_modified.csv \ + --analytics_usa_gov=https://analytics.usa.gov/data/live/sites.csv \ + --censys_snapshot=https://raw.githubusercontent.com/GSA/data/master/dotgov-websites/censys-federal-snapshot.csv \ + --rapid=https://raw.githubusercontent.com/GSA/data/master/dotgov-websites/rdns-federal-snapshot.csv \ + --eot_2012=https://raw.githubusercontent.com/cisagov/scan-target-data/develop/eot-2012.csv \ + --eot_2016=https://raw.githubusercontent.com/cisagov/scan-target-data/develop/eot-2016.csv \ + --cyhy=$OUTPUT_DIR/cyhy_fed_hostnames.csv \ + --other=https://raw.githubusercontent.com/GSA/data/master/dotgov-websites/other-websites.csv cp results/gathered.csv gathered.csv cp results/gathered.csv $OUTPUT_DIR/gathered.csv From f386ac959aa67caf03b51f763b63c9aafe79036d Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 14:33:54 -0400 Subject: [PATCH 237/253] Fix merge error in bump_version.sh --- bump_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bump_version.sh b/bump_version.sh index 81a394f..1d47c62 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -10,7 +10,7 @@ VERSION_FILE=src/version.txt HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" -old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) +old_version=$(cat $VERSION_FILE) if [ $# -ne 1 ] then From c1a4cb12fc2c501a3cb43f5849f16d011622b820 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 14:34:02 -0400 Subject: [PATCH 238/253] Bumping version from 1.3.1 to 1.4.0 --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index 3a3cd8c..88c5fb8 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -1.3.1 +1.4.0 From d2ce4d75eb1a46acf1f698c59042ffa4fcff22a3 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 14:42:10 -0400 Subject: [PATCH 239/253] Change user in container from "gatherer" to "user" --- Dockerfile | 4 ++-- README.md | 2 +- gather-domains.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index cd4f301..0797dbc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,9 +16,9 @@ LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security ARG CISA_GID=421 ARG CISA_UID=${CISA_GID} -ENV CISA_USER="gatherer" +ENV CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} -ENV CISA_HOME="/home/gatherer" +ENV CISA_HOME="/home/cisa" ### # Create unprivileged user diff --git a/README.md b/README.md index bc5256f..b63ed4c 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ This container supports no environment variables. | Mount point | Purpose | |-------------|----------------| -| /home/gatherer/shared | Output | +| /home/cisa/shared | Output | ## Contributing ## diff --git a/gather-domains.sh b/gather-domains.sh index 46c77a5..a45db3a 100755 --- a/gather-domains.sh +++ b/gather-domains.sh @@ -3,7 +3,7 @@ # Gather hostnames and do any necessary scrubbing of the data. ### -HOME_DIR=/home/gatherer +HOME_DIR=/home/cisa OUTPUT_DIR=$HOME_DIR/shared/artifacts # Create the output directory, if necessary From 6952a2d0adbcfa5365f891cbdb4db6f6670122e5 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 15:02:00 -0400 Subject: [PATCH 240/253] Move scripts to src directory --- Dockerfile | 2 +- {scripts => src}/fed_hostnames.py | 0 gather-domains.sh => src/gather-domains.sh | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename {scripts => src}/fed_hostnames.py (100%) rename gather-domains.sh => src/gather-domains.sh (98%) diff --git a/Dockerfile b/Dockerfile index 0797dbc..14cb514 100644 --- a/Dockerfile +++ b/Dockerfile @@ -94,7 +94,7 @@ RUN rm -rf /var/lib/apt/lists/* # Put this just before we change users because the copy (and every # step after it) will always be rerun by docker, but we need to be # root for the chown command. -COPY . ${CISA_HOME} +COPY src ${CISA_HOME} RUN chown -R ${CISA_USER}:${CISA_GROUP} ${CISA_HOME} ### diff --git a/scripts/fed_hostnames.py b/src/fed_hostnames.py similarity index 100% rename from scripts/fed_hostnames.py rename to src/fed_hostnames.py diff --git a/gather-domains.sh b/src/gather-domains.sh similarity index 98% rename from gather-domains.sh rename to src/gather-domains.sh index a45db3a..87fe535 100755 --- a/gather-domains.sh +++ b/src/gather-domains.sh @@ -15,7 +15,7 @@ fi ### # Grab any extra Federal hostnames that CYHY knows about ### -scripts/fed_hostnames.py --output-file=$OUTPUT_DIR/cyhy_fed_hostnames.csv +./fed_hostnames.py --output-file=$OUTPUT_DIR/cyhy_fed_hostnames.csv ### # We need a copy of current-federal since we want to add and remove From a023129d9b053c5f9e95d9bd8f90541dc3cce5c1 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 15:10:00 -0400 Subject: [PATCH 241/253] Remove unused secret --- src/secrets/quote.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 src/secrets/quote.txt diff --git a/src/secrets/quote.txt b/src/secrets/quote.txt deleted file mode 100644 index 93ee1a8..0000000 --- a/src/secrets/quote.txt +++ /dev/null @@ -1 +0,0 @@ -There are no secrets better kept than the secrets everybody guesses. From 206adc53526ae191339b65b758749a6feb8c5d11 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 16:14:13 -0400 Subject: [PATCH 242/253] Add dummy database_creds.yml file as an example --- src/secrets/database_creds.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 src/secrets/database_creds.yml diff --git a/src/secrets/database_creds.yml b/src/secrets/database_creds.yml new file mode 100644 index 0000000..1ce60fa --- /dev/null +++ b/src/secrets/database_creds.yml @@ -0,0 +1,6 @@ +--- +version: '1' + +database: + name: cyhy + uri: mongodb://readonly:the_password@cyhy.example.com:27017/cyhy From 3fa6b6a20b3288e85dc3ee41c21967b3317cb290 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 16:14:56 -0400 Subject: [PATCH 243/253] Update docker-compose file for this project --- docker-compose.yml | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6774387..2a68911 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,37 +1,23 @@ --- version: "3.7" -# This docker-compose file is used to build and test the container +# This docker-compose file is used to build the container secrets: - quote_txt: - file: ./src/secrets/quote.txt + database_creds: + file: ./src/secrets/database_creds.yml services: - example: + gatherer: # Run the container normally build: # VERSION must be specified on the command line: # e.g., --build-arg VERSION=0.0.1 context: . dockerfile: Dockerfile - image: cisagov/example + image: cisagov/gatherer init: true restart: "no" - environment: - - ECHO_MESSAGE=Hello World from docker-compose! - ports: - - target: "8080" - published: "8080" - protocol: tcp - mode: host secrets: - - source: quote_txt - target: quote.txt - - example-version: - # Run the container to collect version information - image: cisagov/example - init: true - restart: "no" - command: --version + - source: database_creds + target: database_creds.yml From fba3c1bbd4404b022f9d5dd91f545d5b6b4c2527 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 16:39:50 -0400 Subject: [PATCH 244/253] Update service name in tests --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 28d6c42..a3dcb85 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ # Third-Party Libraries import pytest -MAIN_SERVICE_NAME = "example" +MAIN_SERVICE_NAME = "gatherer" VERSION_SERVICE_NAME = f"{MAIN_SERVICE_NAME}-version" From ea50e66fa8327dbb6429df4907c14f68c6cb3e4d Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 16:48:03 -0400 Subject: [PATCH 245/253] Add redis to example composition --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 2a68911..535c637 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,9 +15,13 @@ services: # e.g., --build-arg VERSION=0.0.1 context: . dockerfile: Dockerfile + depends_on: + - redis image: cisagov/gatherer init: true restart: "no" secrets: - source: database_creds target: database_creds.yml + redis: + image: redis:alpine From d0342b2a994785eee223c23a503b1172c13c397f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 18 May 2021 17:09:57 -0400 Subject: [PATCH 246/253] Comment out some tests The container doesn't currently support the functionality being checked by the tests. See #57 for more details. --- tests/conftest.py | 17 ++--- tests/container_test.py | 142 +++++++++++++++++++++------------------- 2 files changed, 83 insertions(+), 76 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a3dcb85..847765e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,14 +16,15 @@ def main_container(dockerc): return dockerc.containers(service_names=[MAIN_SERVICE_NAME], stopped=True)[0] -@pytest.fixture(scope="session") -def version_container(dockerc): - """Return the version container from the docker composition. - - The version container should just output the version of its underlying contents. - """ - # find the container by name even if it is stopped already - return dockerc.containers(service_names=[VERSION_SERVICE_NAME], stopped=True)[0] +# See #57 +# @pytest.fixture(scope="session") +# def version_container(dockerc): +# """Return the version container from the docker composition. + +# The version container should just output the version of its underlying contents. +# """ +# # find the container by name even if it is stopped already +# return dockerc.containers(service_names=[VERSION_SERVICE_NAME], stopped=True)[0] def pytest_addoption(parser): diff --git a/tests/container_test.py b/tests/container_test.py index 6153028..4adaa40 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -3,10 +3,10 @@ # Standard Python Libraries import os -import time -# Third-Party Libraries -import pytest +# import time + +# import pytest ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker-compose!" @@ -26,68 +26,74 @@ def test_container_count(dockerc): ), "Wrong number of containers were started." -def test_wait_for_ready(main_container): - """Wait for container to be ready.""" - TIMEOUT = 10 - for i in range(TIMEOUT): - if READY_MESSAGE in main_container.logs().decode("utf-8"): - break - time.sleep(1) - else: - raise Exception( - f"Container does not seem ready. " - f'Expected "{READY_MESSAGE}" in the log within {TIMEOUT} seconds.' - ) - - -def test_wait_for_exits(main_container, version_container): - """Wait for containers to exit.""" - assert main_container.wait() == 0, "Container service (main) did not exit cleanly" - assert ( - version_container.wait() == 0 - ), "Container service (version) did not exit cleanly" - - -def test_output(main_container): - """Verify the container had the correct output.""" - main_container.wait() # make sure container exited if running test isolated - log_output = main_container.logs().decode("utf-8") - assert SECRET_QUOTE in log_output, "Secret not found in log output." - - -@pytest.mark.skipif( - RELEASE_TAG in [None, ""], reason="this is not a release (RELEASE_TAG not set)" -) -def test_release_version(): - """Verify that release tag version agrees with the module version.""" - pkg_vars = {} - with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] - assert ( - RELEASE_TAG == f"v{project_version}" - ), "RELEASE_TAG does not match the project version" - - -def test_log_version(version_container): - """Verify the container outputs the correct version to the logs.""" - version_container.wait() # make sure container exited if running test isolated - log_output = version_container.logs().decode("utf-8").strip() - pkg_vars = {} - with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] - assert ( - log_output == project_version - ), f"Container version output to log does not match project version file {VERSION_FILE}" - - -def test_container_version_label_matches(version_container): - """Verify the container version label is the correct version.""" - pkg_vars = {} - with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] - assert ( - version_container.labels["org.opencontainers.image.version"] == project_version - ), "Dockerfile version label does not match project version" +# See #57 +# def test_wait_for_ready(main_container): +# """Wait for container to be ready.""" +# TIMEOUT = 10 +# for i in range(TIMEOUT): +# if READY_MESSAGE in main_container.logs().decode("utf-8"): +# break +# time.sleep(1) +# else: +# raise Exception( +# f"Container does not seem ready. " +# f'Expected "{READY_MESSAGE}" in the log within {TIMEOUT} seconds.' +# ) + + +# See #57 +# def test_wait_for_exits(main_container, version_container): +# """Wait for containers to exit.""" +# assert main_container.wait() == 0, "Container service (main) did not exit cleanly" +# assert ( +# version_container.wait() == 0 +# ), "Container service (version) did not exit cleanly" + + +# See #57 +# def test_output(main_container): +# """Verify the container had the correct output.""" +# main_container.wait() # make sure container exited if running test isolated +# log_output = main_container.logs().decode("utf-8") +# assert SECRET_QUOTE in log_output, "Secret not found in log output." + + +# See #57 +# @pytest.mark.skipif( +# RELEASE_TAG in [None, ""], reason="this is not a release (RELEASE_TAG not set)" +# ) +# def test_release_version(): +# """Verify that release tag version agrees with the module version.""" +# pkg_vars = {} +# with open(VERSION_FILE) as f: +# exec(f.read(), pkg_vars) # nosec +# project_version = pkg_vars["__version__"] +# assert ( +# RELEASE_TAG == f"v{project_version}" +# ), "RELEASE_TAG does not match the project version" + + +# See #57 +# def test_log_version(version_container): +# """Verify the container outputs the correct version to the logs.""" +# version_container.wait() # make sure container exited if running test isolated +# log_output = version_container.logs().decode("utf-8").strip() +# pkg_vars = {} +# with open(VERSION_FILE) as f: +# exec(f.read(), pkg_vars) # nosec +# project_version = pkg_vars["__version__"] +# assert ( +# log_output == project_version +# ), f"Container version output to log does not match project version file {VERSION_FILE}" + + +# See #57 +# def test_container_version_label_matches(version_container): +# """Verify the container version label is the correct version.""" +# pkg_vars = {} +# with open(VERSION_FILE) as f: +# exec(f.read(), pkg_vars) # nosec +# project_version = pkg_vars["__version__"] +# assert ( +# version_container.labels["org.opencontainers.image.version"] == project_version +# ), "Dockerfile version label does not match project version" From 6ccaae2acd00bbf1b018148ce83f2f09629fae3c Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 19 May 2021 09:48:10 -0400 Subject: [PATCH 247/253] Update repo names in skeleton file --- CONTRIBUTING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4222005..df396a2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ all of which should be in this repository. If you want to report a bug or request a new feature, the most direct method is to [create an -issue](https://github.com/cisagov/skeleton-docker/issues) in this +issue](https://github.com/cisagov/gatherer/issues) in this repository. We recommend that you first search through existing issues (both open and closed) to check if your particular issue has already been reported. If it has then you might want to add a comment @@ -25,7 +25,7 @@ one. ## Pull requests ## If you choose to [submit a pull -request](https://github.com/cisagov/skeleton-docker/pulls), you will +request](https://github.com/cisagov/gatherer/pulls), you will notice that our continuous integration (CI) system runs a fairly extensive set of linters and syntax checkers. Your pull request may fail these checks, and that's OK. If you want you can stop there and @@ -111,9 +111,9 @@ can create and configure the Python virtual environment with these commands: ```console -cd skeleton-docker -pyenv virtualenv skeleton-docker -pyenv local skeleton-docker +cd gatherer +pyenv virtualenv gatherer +pyenv local gatherer pip install --requirement requirements-dev.txt ``` From 9122873d26005a4456840a166dbe06b9a51cd576 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 19 May 2021 10:06:02 -0400 Subject: [PATCH 248/253] Update lineage config to point to the correct upstream repo --- .github/lineage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/lineage.yml b/.github/lineage.yml index 8dfc20b..b10c80c 100644 --- a/.github/lineage.yml +++ b/.github/lineage.yml @@ -3,4 +3,4 @@ version: "1" lineage: skeleton: - remote-url: https://github.com/cisagov/skeleton-generic.git + remote-url: https://github.com/cisagov/skeleton-docker.git From 3709be9852ff15c3bf28c94a13e303dfa6b5eeed Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 19 May 2021 10:07:44 -0400 Subject: [PATCH 249/253] Pare down codeowners to only those folks who are knowledgeable about this repo --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9c3d21f..ab07ea9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,8 +3,8 @@ # These owners will be the default owners for everything in the # repo. Unless a later match takes precedence, these owners will be # requested for review when someone opens a pull request. -* @dav3r @felddy @hillaryj @jsf9k @mcdonnnj +* @dav3r @jsf9k @mcdonnnj # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. -/.github/ @dav3r @felddy @hillaryj @jsf9k @mcdonnnj +/.github/ @dav3r @felddy @jsf9k @mcdonnnj From 9acb5910ae4b0eb5a44e0b1549df011490dde195 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Wed, 19 May 2021 14:55:33 -0400 Subject: [PATCH 250/253] Fix place where I neglected to update the Docker image name Co-authored-by: dav3r --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ce70bf5..dbb6cc1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,7 @@ on: env: BUILDX_CACHE_DIR: ~/.cache/buildx - IMAGE_NAME: cisagov/example + IMAGE_NAME: cisagov/gatherer PIP_CACHE_DIR: ~/.cache/pip PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,\ linux/arm64,linux/ppc64le,linux/s390x" From 5d6fe225b0c7bed330ca568ab025502daf9906ef Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Wed, 19 May 2021 15:02:57 -0400 Subject: [PATCH 251/253] Capitalize title Co-authored-by: dav3r --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b63ed4c..c8058c7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# gatherer # +# Gatherer # [![GitHub Build Status](https://github.com/cisagov/gatherer/workflows/build/badge.svg)](https://github.com/cisagov/gatherer/actions) [![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/gatherer.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/gatherer/alerts/) From 204b67e5c09daca181a19955bce3c8e48702953a Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Wed, 19 May 2021 15:39:11 -0400 Subject: [PATCH 252/253] Add a TODO Co-authored-by: dav3r --- tests/container_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/container_test.py b/tests/container_test.py index 4adaa40..e19fa5c 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,5 +1,7 @@ #!/usr/bin/env pytest -vs """Tests for example container.""" +# TODO: Make container tests functional +# See https://github.com/cisagov/gatherer/issues/57 # Standard Python Libraries import os From a1aa4f6b76778b2c571915e60d611203a93bbcd1 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 20 May 2021 10:04:45 -0400 Subject: [PATCH 253/253] Add a note explaining why the secrets in src/secrets are present Co-authored-by: Dave Redmin --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c8058c7..dc34422 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,12 @@ a precursor to scanning by [pshtt](https://github.com/cisagov/pshtt), This Docker container is intended to be run via [orchestrator](https://github.com/cisagov/orchestrator). +__N.B.:__ The secrets in the `src/secrets` directory are only used +when testing via the `docker-compose.yml` composition. Normally this +Docker container is run via the Docker composition in +[cisagov/orchestrator](https://github.com/cisagov/orchestrator), which +expects the secrets in a different location. + ## Usage ## ### Install ###