feat: Support for SSP writing with jinja templating. (#787)

* chore: Committing to change priorities.

Signed-off-by: Chris Butler <[email protected]>

* fix: Demonstration of jinja extensions allowing processing of oscal fields.

Signed-off-by: Chris Butler <[email protected]>

* fix:Added example on jinja if tags

Signed-off-by: Chris Butler <[email protected]>

* feat: Add Jinja markdown filters

Signed-off-by: Chris Butler <[email protected]>

* feat: Adding jinja transform command

Signed-off-by: Chris Butler <[email protected]>

* chore: Lazy stashing.

Signed-off-by: Chris Butler <[email protected]>

* feat: updated signatures

Signed-off-by: Chris Butler <[email protected]>

* feat: Adding kwarg to allow jinja templates to adjust header level

Signed-off-by: Chris Butler <[email protected]>

* chore: Add ssp io (#938)

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: Fix control response

Signed-off-by: Ekaterina Nikonova <[email protected]>

* fix: Corrected mdtags to use kwargs properly

Signed-off-by: Chris Butler <[email protected]>

* chore: Adjust control response

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: Remove extra test data

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: Fix test name

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: Small adjustment

Signed-off-by: Ekaterina Nikonova <[email protected]>

* fix: Working ssp templating

Signed-off-by: Chris Butler <[email protected]>

* chore: adding missing test files.

Signed-off-by: Chris Butler <[email protected]>

* chore: Print response for this system

Signed-off-by: Ekaterina Nikonova <[email protected]>

* fix: Minor edits

Signed-off-by: Chris Butler <[email protected]>

* fix: CICD Cleanups (#943)

Signed-off-by: Chris Butler <[email protected]>

* chore: Adjust tables and headers (#944)

Signed-off-by: Ekaterina Nikonova <[email protected]>

* fix: Correcting tables in ssp_io

Signed-off-by: Chris Butler <[email protected]>

* fix: fix attribute error in catalog_interface (#946)

Signed-off-by: Doug Chivers <[email protected]>

Co-authored-by: Doug Chivers <[email protected]>

* chore: Fix tests and tables

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: Add more tests

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: Small fix

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: documentation for the jinja command. (#948)

* fix: Added jinja cmd documentation

Signed-off-by: Chris Butler <[email protected]>

* fix: Revising docs based on PR feedback.

Signed-off-by: Chris Butler <[email protected]>

* chore: More tests

Signed-off-by: Ekaterina Nikonova <[email protected]>

* fix: add trailing newlines to md front_matter content (#956)

fixes format issue in pandoc docx conversion of front matter, where the header of the following section is included in the previous paragraph if there is no trailing newline in the markdown file of the previous paragraph.

Signed-off-by: Doug Chivers <[email protected]>

Co-authored-by: Doug Chivers <[email protected]>

* fix: Removed extra unnessary unit tests and unneeded code. (#959)

Signed-off-by: Chris Butler <[email protected]>

* chore: Add Jinja (#958)

* chore: Add jinja tests

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: Address review feedback

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: Address feedback

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: Small fix

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: Fix tests

Signed-off-by: Ekaterina Nikonova <[email protected]>

* chore: Small enchancement

Signed-off-by: Ekaterina Nikonova <[email protected]>

* fix: update control title format for ssp output (#960)

Signed-off-by: Doug Chivers <[email protected]>

Co-authored-by: Doug Chivers <[email protected]>

* chore: Adjust title in ssp

Signed-off-by: Ekaterina Nikonova <[email protected]>

Co-authored-by: Frank Suits <[email protected]>
Co-authored-by: Ekaterina Nikonova <[email protected]>
Co-authored-by: Ekaterina Nikonova <[email protected]>
Co-authored-by: D9 <[email protected]>
Co-authored-by: Doug Chivers <[email protected]>

.github/workflows/conventional-pr.yml

@@ -1,4 +1,4 @@
-name: "Lint PR"
+name: "Lint PR title"
 
 on:
   pull_request_target:
@@ -8,6 +8,7 @@ on:
       - synchronize
     branches:
       - 'develop'
+      - 'main'
 jobs:
   lint:
     if: ${{ github.head_ref != 'develop' }}

.github/workflows/python-push.yml

@@ -79,12 +79,6 @@ jobs:
       if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.7' }}
       run: |
         make test-cov
-    - name: Push code-cov
-      if: ${{ !(matrix.os == 'ubuntu-latest' && matrix.python-version == '3.7') }}
-      uses: codecov/codecov-action@v1
-      with: 
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: ./coverage.xml
   deploy:
 
     runs-on: ubuntu-latest

.github/workflows/python-test.yml

@@ -10,9 +10,6 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - name: Don't mess with line endings
-      run: |
-        git config --global core.autocrlf false
     - name: Don't mess with line endings
       run: |
         git config --global core.autocrlf false

.pre-commit-config.yaml

@@ -73,7 +73,7 @@ repos:
     rev: 0.7.11
     hooks:
     - id: mdformat
-      exclude: "CHANGELOG.md|docs/mkdocs_code_of_conduct.md|docs/api_reference|tests/data/author|docs/contributing/mkdocs_contributing.md"
+      exclude: "CHANGELOG.md|docs/mkdocs_code_of_conduct.md|docs/api_reference|tests/data/author|docs/contributing/mkdocs_contributing.md|tests/data/jinja_markdown_include"
       additional_dependencies:
       - mdformat-tables
       - mdformat-config

docs/api_reference/trestle.core.commands.author.jinja.md

@@ -0,0 +1,2 @@
+::: trestle.core.commands.author.jinja
+handler: python

docs/api_reference/trestle.core.jinja.md

@@ -0,0 +1,2 @@
+::: trestle.core.jinja
+handler: python

docs/api_reference/trestle.core.ssp_io.md

@@ -0,0 +1,2 @@
+::: trestle.core.ssp_io
+handler: python

docs/assets/sample_ssp.md.jinja

@@ -0,0 +1,37 @@
+# SSP template 
+<!--
+This produces an SSP analog. 
+-->
+
+# Frontmatter
+<!-- This includes frontmatter.md showing how arbitrary prose can be inserted into a document --> 
+{% md_clean_include 'frontmatter.md' heading_level=2 %}
+
+
+
+
+
+
+<!-- This iterates over the controls in a resolved catalog and prints out the control statement, summary table,
+and implemented requirements in a format similar to a FedRAMP .docx
+
+Note that the catalog interface has controls properly sorted so is useful to present the controls how they are organised
+in the original catalog 
+-->
+# Control responses and guidance
+{% for group in catalog_interface.get_all_groups_from_catalog() +%}
+## {{ group.title }} {{ group.class }} \({{ group.id|upper }}\)
+
+{% for control in catalog_interface._control_dict.values() %}
+{% if control.group_id ==  group.id %}
+{{ ssp_md_writer.get_control_statement(control.control.id, 2) }}
+
+
+#### {{ catalog_interface.get_label(control.control) }} Summary information
+{{ ssp_md_writer.get_fedramp_control_tables(control.control.id, 4) }}
+
+#### What is the solution and how is it implemented?
+{{ ssp_md_writer.get_control_response(control.control.id, 4)}}
+{% endif %}
+{% endfor %}
+{% endfor %}

docs/trestle_author_jinja.md

@@ -0,0 +1,118 @@
+# Trestle author jinja - output templating support for oscal documents.
+
+Unfortunately OSCAL documents are not yet universally accepted. Therefore to support various OSCAL and non-OSCAL compliance workflows `trestle author jinja` is designed to provide end users with the ability to use jinja to produce customized output. This complements the more structured commands `trestle author catalog-{assemble|generate}`, `trestle author profile-{assemble|generate}` and `trestle author ssp-{assemble|generate}` and allows arbitrary use of jinja.
+
+## Jinja and jinja extensions provided by trestle.
+
+[Jinja](https://jinja.palletsprojects.com/en/3.0.x/) is a powerful templating engine that is both more flexible that pure 'Moustache' approaches, and not coupled to a particular web application server (as an example Django templates). Users are encouraged to review the [template designer documentation](https://jinja.palletsprojects.com/en/3.0.x/templates/) for jinja as all core functionality is exposed.
+
+Trestle's implementation of the Jinja command works in the following way:
+
+1. The template search space, by default, is relative to the current working directory.
+1. Trestle can inject a lookup table, into the jinja variables to contain booleans / substitutions required by end users using Moustache style variable substitutions.
+1. Trestle can provide a number of interfaces to OSCAL objects, currently a resolved catalog and a SSP, into jinja.
+1. Trestle supports custom jinja tags for importing.
+1. The jinja templating is recursive, to ensure all jinja tags are resolved as appropriate.
+
+More details will be on each of these points below.
+
+## CLI invocation
+
+Note the examples here use markdown, however, jinja can quite easily target xml or html if used w/o specific markdown content.
+`trestle author jinja -i input_template.md.jinja -o output_file.md -ssp SSP_NAME -p PROFILE_NAME -lut lookup_table.yaml -elp lut.prefix`
+
+- `-i` input file path, relative to cwd. Users are encouraged to use the `file_name.target_extension.jinja` best practice as it helps mitigate issues, however is not required.
+- `-o` final output path, relative to cwd.
+- `-ssp` (optional) ssp name (in the trestle project). When used the jinja template will have a `ssp_md_writer` variable exposed to use.
+- `-p` (optional) profile name (in the trestle project). When used the jinja template will have `resolved_catalog` and `catalog_interface` variables to use.
+- `-lut` (optional) loads yaml into a dictionary in python for which each (top level) variable is available in jinja.
+- `-elp` (optional) a period separated prefix for the variables in the lookup table. E.g. if the lut contained `banana: yellow` and the prefix was `fruit.tropical` using `{{ fruit.tropical.banana }}` would print out `yellow` in the jinja template.
+
+## Sample jinja templates
+
+Note in addition to these templates some testing examples are available [here](https://github.com/IBM/develop)
+
+| template                                          | Description                                                                                          | Optional args required                                                                                          |
+| ------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- |
+| [Simple SSP template](assets/sample_ssp.md.jinja) | Sample ssp jinja template which prints out all control responses and includes a front-matter section | `-ssp` and `-p` for the ssp json and corresponding profile, respectively. Also requires a `frontmatter.md` file |
+
+## Variable availability in the jinja template.
+
+### LUT
+
+The lookup table is primarily used for string substitution and to provide variables for basic logic operations in jinja e.g.
+
+The LUT:
+
+```yaml
+names:
+  OSCAL: Open Security Compliance Assessment Language
+  trestle_pip: compliance-trestle
+  trestle_module: trestle
+variables:
+  mac_os: true
+```
+
+The Jinja template:
+
+```jinja
+
+Install via pip install {{ names.trestle_pip }} and invoke at the python REPL by import {{ names.trestle_module }}
+
+{% if variables.mac_os %}
+Users are recommended to use homebrew to install the latest python 3 and then install python within a venv.
+{% endif %}
+```
+
+The output:
+
+```text
+
+Install via pip install compliance-trestle and invoke at the python REPL by import trestle
+
+Users are recommended to use homebrew to install the latest python 3 and then install python within a venv.
+```
+
+Users are free to use the LUT to inject more complex variables (arrays of data etc) to use at their own will using standard jinja templating.
+
+### Resolved catalog interface (profile)
+
+Passing `-p` exposes a catalog, resolved from the profile, at `catalog` and a `trestle.core.catalog_interface.CatalogInterface` at `catalog_interface`.
+
+This allows user to perform various task such as iterating ove reach group and printing the group title.
+
+```jinja
+{% for group in catalog_interface.get_all_groups_from_catalog() +%}
+## {{ group.title }} {{ group.class }} \({{ group.id|upper }}\)
+```
+
+### SSP interface.
+
+If `-ssp` is passed a variable within the jinja template called `ssp_md_writer` is made available which is an instance of `trestle.core.ssp_io.SSPMarkdownWriter`.
+`-ssp` requires that `-p` has also been set.
+
+This as allows users, as an example to print out a control response, as markdown
+
+```jinja
+#### What is the solution and how is it implemented?
+{{ ssp_md_writer.get_control_response('my_control_id', 3)}}
+```
+
+## Custom Jinja tags.
+
+Trestle provides to custom jinja tags for use specifically with markdown: `mdsection_include` and `md_clean_include`.
+
+`md_clean_include` is similar to the native `{% include 'sub_template' %}` that jinja provides except for the following:
+
+1. `md_clean_include` will look for yaml headers in the markdown content and exclude it from the template
+1. `md_clean_include` can be used with an optional keyword argument `heading_level` argument
+   1. `{% md_clean_include 'path_to_file.md' heading_level=2 %}`
+   1. The heading level argument adjusts to (based on the number of hashes) the most significant heading in the document, if headings exist.
+
+`mdsection_include` is similar to the native `md_clean_include` except that.:
+
+1. `mdsection_include` requires an second positional argument which is the title of a heading, from a markdown file, which you want the cotent from.
+   1. E.g:  `{% mdsection_include 'test_markdown.md' '# Header we want' %}`
+1. `mdsection_include` can be used with an optional keyword argument `heading_level` argument similar to `md_clean_include`
+   1. `{% mdsection_include 'test_markdown.md' '# Header we want' %}`
+   1. The heading level argument adjusts to (based on the number of hashes) the most significant heading in the chosen section, if headings exist.

docs/tutorials/ssp_profile_catalog_authoring/ssp_profile_catalog_authoring.md

@@ -173,7 +173,7 @@ Example usage for creation of the markdown:
 
 In this example the profile has previously been imported into the trestle project directory.  The profile itself must be in the trestle directory, but the imported catalogs and profiles may be URI's with href's as described below.
 
-The `-s --section` argument specifies the name of Parts in the control for which the corresponding prose should be included in the control's markdown file.  Each colon-separated pair refers to the actual part name first, followed by the form that should be used in the heading for that section.  This is done because the name itself may be abbreviated and lack needed spaces between words.
+The `-s --section` argument specifies the name of Parts in the control for which the corresponding prose should be included in the control's markdown file.  Each colon-separated pair refers to the actual part name first, followed by the form that should be used in the heading for that section.  This is done because the name itself may be abbreviated and lack needed spaces between words. If no section labels are provided all parts are included using the default 'name' as specified in the underlying OSCAL.
 
 (Note that the single quotes are required on Unix-like systems, but on Windows they are only needed if the contained string includes spaces, in which case *double* quotes should be used.)
 

mkdocs.yml

@@ -31,6 +31,7 @@ nav:
 - Trestle command-line interface (CLI):
   - CLI for OSCAL documents: cli.md
   - CLI for authoring governed content: trestle_author.md
+  - CLI for jinja template processing: trestle_author_jinja.md
 - Contributing:
   - Code of Conduct: mkdocs_code_of_conduct.md
   - Contributing overview: contributing/mkdocs_contributing.md
@@ -64,6 +65,7 @@ nav:
           - docs: api_reference/trestle.core.commands.author.docs.md
           - folders: api_reference/trestle.core.commands.author.folders.md
           - headers: api_reference/trestle.core.commands.author.headers.md
+          - jinja: api_reference/trestle.core.commands.author.jinja.md
           - profile: api_reference/trestle.core.commands.author.profile.md
           - ssp: api_reference/trestle.core.commands.author.ssp.md
           - versioning:
@@ -93,6 +95,7 @@ nav:
       - duplicates_validator: api_reference/trestle.core.duplicates_validator.md
       - err: api_reference/trestle.core.err.md
       - generators: api_reference/trestle.core.generators.md
+      - jinja: api_reference/trestle.core.jinja.md
       - markdown:
         - markdown_api: api_reference/trestle.core.markdown.markdown_api.md
         - markdown_const: api_reference/trestle.core.markdown.markdown_const.md
@@ -115,6 +118,7 @@ nav:
       - remote:
         - cache: api_reference/trestle.core.remote.cache.md
       - repository: api_reference/trestle.core.repository.md
+      - ssp_io: api_reference/trestle.core.ssp_io.md
       - trestle_base_model: api_reference/trestle.core.trestle_base_model.md
       - utils: api_reference/trestle.core.utils.md
       - validator: api_reference/trestle.core.validator.md

sonar-project.properties

@@ -1,4 +1,4 @@
 sonar-organisation=compliance-trestle
 sonar-projectKey=compliance-trestle
 
-sounar-sources=.
+sonar-sources=.