CI/CD Script Refactoring and Enhancements (usnistgov#479)

* refactored unit tests to improve performance and output; added XML Schema validation for generated schema * fixed broken link detection in repo Markdown files * Added optimization to limit published artifacts for generated docs
ohsh6o · Aug 29, 2019 · 7eacfc9 · 7eacfc9
1 parent 26dfd1c
commit 7eacfc9
Show file tree

Hide file tree

Showing 24 changed files with 4,174 additions and 441 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -23,12 +23,13 @@ references:
   restore_gem_cache: &restore_gem_cache
     restore_cache:
       keys:
-        - v2-gem-cache-{{ arch }}-{{ checksum "git-oscal/docs/Gemfile.lock" }}
+        - gem-cache-v9-{{ arch }}-{{ checksum "git-oscal/docs/Gemfile.lock" }}
+        - gem-cache-v9-{{ arch }}-
   save_gem_cache: &save_gem_cache
     save_cache:
-      key: v2-gem-cache-{{ arch }}-{{ checksum "git-oscal/docs/Gemfile.lock" }}
+      key: gem-cache-v9-{{ arch }}-{{ checksum "git-oscal/docs/Gemfile.lock" }}
       paths:
-        - ~/jekyll-bundle
+        - git-oscal/docs/vendor/bundle
 commands:
   install-schematron:
     description: "Install the schematron skeleton"
@@ -142,9 +143,9 @@ commands:
           name: Install Bundler
           command: |
             cd "$OSCAL_REPO_DIR/docs"
-            echo gem install bundler
-            bundle install --path ~/jekyll-bundle
-            gem install html-proofer
+            # gem install bundler
+            # gem install html-proofer
+            bundle install --path vendor/bundle --jobs=4 --retry=3 && bundle clean
   check-skip-build:
     steps:
       - run:
@@ -204,7 +205,7 @@ jobs: # a collection of
       - run:
           name: Validate metaschema instances
           command: |
-            bash "$CICD_DIR/validate-metaschema.sh" "$OSCAL_BUILD_DIR"
+            bash "$CICD_DIR/validate-metaschema.sh" -w "$OSCAL_BUILD_DIR" --scratch-dir "$OSCAL_BUILD_DIR"
       - *publish_artifacts
       - persist_to_workspace:
           root: build_artifacts
@@ -217,10 +218,11 @@ jobs: # a collection of
       - *attach_build_workspace
       - check-skip-build
       - install-maven-dependencies
+      - install-xmllint
       - run:
           name: Generate OSCAL schemas
           command: |
-            bash "$CICD_DIR/generate-schema.sh" "$OSCAL_BUILD_DIR"
+            bash "$CICD_DIR/generate-schema.sh" -w "$OSCAL_BUILD_DIR"
       - *publish_artifacts
       - persist_to_workspace:
           root: build_artifacts
@@ -237,7 +239,7 @@ jobs: # a collection of
       - run:
           name: Generate OSCAL schemas
           command: |
-            bash "$CICD_DIR/generate-content-converters.sh" "$OSCAL_BUILD_DIR"
+            bash "$CICD_DIR/generate-content-converters.sh" -w "$OSCAL_BUILD_DIR"
       - *publish_artifacts
       - persist_to_workspace:
           root: build_artifacts
@@ -256,7 +258,7 @@ jobs: # a collection of
       - run:
           name: Generate OSCAL converters
           command: |
-            bash "$CICD_DIR/copy-and-convert-content.sh" "$OSCAL_BUILD_DIR"
+            bash "$CICD_DIR/copy-and-convert-content.sh" -w "$OSCAL_BUILD_DIR"
       - *publish_artifacts
       - persist_to_workspace:
           root: build_artifacts
@@ -270,9 +272,10 @@ jobs: # a collection of
       - check-skip-build
       - install-markdown-link-check
       - run:
-          name: Validate content instances
+          name: Validate repo Markdown content instances
           command: |
-            find "$OSCAL_REPO_DIR" -path ./docs -prune -o -name \*.md -exec markdown-link-check -q {} \;
+            # this command will filter out any docs Markdown files, which are checked in a different job
+            find "$OSCAL_REPO_DIR" -path "${OSCAL_REPO_DIR}/docs" -prune -o -name \*.md -print0 | xargs -0 -n1 markdown-link-check -q
   job-validate-content:
     executor: java-executor
     steps:
@@ -284,7 +287,7 @@ jobs: # a collection of
       - run:
           name: Validate content instances
           command: |
-            bash "$CICD_DIR/validate-content.sh" "$OSCAL_BUILD_DIR"
+            bash "$CICD_DIR/validate-content.sh" -w "$OSCAL_BUILD_DIR"
   job-roundtrip-conversions:
     executor: python-executor
     steps:
@@ -298,7 +301,7 @@ jobs: # a collection of
       - run:
           name: Generate round trip XML and JSON conversions
           command: |
-            bash "${CICD_DIR}/validate-round-trips.sh" "$OSCAL_BUILD_DIR"
+            bash "${CICD_DIR}/validate-content-conversion-round-trips.sh" -w "$OSCAL_BUILD_DIR" --scratch-dir "$OSCAL_BUILD_DIR"
       - *publish_artifacts
   job-generate-docs:
     executor: java-executor
@@ -310,15 +313,16 @@ jobs: # a collection of
       - run:
           name: Generate schema documentation
           command: |
-            #cp -r "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
-            bash -x "$CICD_DIR/generate-model-documentation.sh" "$OSCAL_BUILD_DIR"
-            # echo "-- fix the fact that we cannot generate directly to the build directory due to a bug"
-            # mv docs "$OSCAL_BUILD_DIR"/docs
+            bash "$CICD_DIR/generate-model-documentation.sh" -w "$OSCAL_BUILD_DIR"
+      - run:
+          name: Compress built docs
+          command: |
+            tar -zcvf "${OSCAL_BUILD_DIR}/docs.tar.gz" --remove-files "${OSCAL_BUILD_DIR}/docs"
       - *publish_artifacts
       - persist_to_workspace:
           root: build_artifacts
           paths:
-            - docs
+            - docs.tar.gz
   job-deploy-artifacts:
     executor: java-executor
     steps: # a collection of executable commands
@@ -370,21 +374,21 @@ jobs: # a collection of
       - check-skip-build
       - *restore_gem_cache
       - install-bundler
+      - run:
+          name: Restore generated site content
+          command: |
+            # restore docs from tar
+            tar -zxvf "${OSCAL_BUILD_DIR}/docs.tar.gz" --directory "${OSCAL_REPO_DIR}/docs"
       - run:
           name: Build and Validate Jekyll site
           command: |
-            cp -vr "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
+            # cp -vr "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
             cd "$OSCAL_REPO_DIR/docs"
             bundle exec jekyll build -d _site/OSCAL
-            htmlproofer --allow-hash-href --url-ignore "/\/OSCAL\/docs\/schemas\//,/#/,/https:\/\/raw\.githubusercontent\.com/" --assume-extension --log-level :debug ./_site
+            bundle exec htmlproofer --allow-hash-href --url-ignore "/\/OSCAL\/docs\/schemas\//,/#/,/https:\/\/raw\.githubusercontent\.com/" --assume-extension --log-level :debug ./_site
           environment:
             JEKYLL_ENV: production
       - *save_gem_cache
-      - run:
-          name: Compress build docs
-          command: |
-            tar -zcvf "${OSCAL_BUILD_DIR}/docs.tar.gz" "${OSCAL_BUILD_DIR}/docs"
-            rm -rf "${OSCAL_BUILD_DIR}/docs"
       - *publish_artifacts
   job-build-and-deploy-website:
     executor: ruby-node-executor
@@ -394,10 +398,15 @@ jobs: # a collection of
       - check-skip-build
       - *restore_gem_cache
       - install-bundler
+      - run:
+          name: Restore generated site content
+          command: |
+            # restore docs from tar
+            tar -zxvf "${OSCAL_BUILD_DIR}/docs.tar.gz" --directory "${OSCAL_REPO_DIR}/docs"
       - run:
           name: Build jekyll site
           command: |
-            cp -vr "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
+            # cp -vr "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR"
             cd "$OSCAL_REPO_DIR/docs"
             bundle exec jekyll build
             mkdir -p _site/.circleci
@@ -421,7 +430,7 @@ jobs: # a collection of
       - run:
           name: Package Release
           command: |
-            bash -x "$CICD_DIR/package-release.sh" "$OSCAL_BUILD_DIR"
+            bash "$CICD_DIR/package-release.sh" "$OSCAL_BUILD_DIR"
 workflows:
   version: 2
   build:

diff --git a/.github/ISSUE_TEMPLATE/README.md b/.github/ISSUE_TEMPLATE/README.md
@@ -2,7 +2,6 @@
 
 This directory contains templates for creating new OSCAL issues in GitHub. The structure and contents of the directory are as follows:
 
-* [defect-template.md](defect-template.md): This file contains a template for creating a new issue in GitHub to report an OSCAL bug.
-* [issue-template.md](issue-template.md): This file contains a template for creating a generic new issue in GitHub (an issue that doesn't fit into any of the other categories).
-* [question_template.md](question_template.md): This file contains a template for creating a new issue in GitHub containing a general question about OSCAL.
-* [user-story_template.md](user-story_template.md): This file contains a template for creating a new issue in GitHub containing a user story.
+* [bug_report.md](bug_report.md): This file contains a template for creating a new issue in GitHub to report an OSCAL bug.
+* [question.md](question.md): This file contains a template for creating a new issue in GitHub containing a general question about OSCAL.
+* [feature_request.md](feature_request.md): This file contains a template for creating a new issue in GitHub containing a user story.
diff --git a/build/ci-cd/README.md b/build/ci-cd/README.md
@@ -17,10 +17,10 @@ The primary CircleCI supported CI/CD workflow, ```build```, is uses a series of
 1. [validate-content.sh](validate-content.sh): Validates OSCAL content in the repository's [src](https://github.com/usnistgov/OSCAL/tree/master/src) against the content's respective OSCAL model and format.
 1. [generate-content-converters.sh](generate-content-converters.sh): Generates [JSON to XML](https://github.com/usnistgov/OSCAL/tree/master/xml/convert) and [XML to JSON](https://github.com/usnistgov/OSCAL/tree/master/json/convert) content conversion scripts for each OSCAL model based on their [respective metaschemas](https://github.com/usnistgov/OSCAL/tree/master/src/metaschema).
 1. [copy-and-convert-content.sh](copy-and-convert-content.sh): Copies selective content from the [src/content](https://github.com/usnistgov/OSCAL/tree/master/src/content) directory, and generates instance of this content in alternate OSCAL formats using the appropriate content converters. Note __This script requires that the process have write access to the Git repository branch that the CI/CD is operating on to work completely.__
-1. [validate-round-trips.sh](validate-round-trips.sh): Performs round trip validations of the content from XML->JSON->XML to confirm that the conversions are lossless and that resulting files comply with their schemas.
+1. [validate-content-conversion-round-trips.sh](validate-content-conversion-round-trips.sh): Performs round trip validations of the content from XML->JSON->XML to confirm that the conversions are lossless and that resulting files comply with their schemas.
 
 Additional scripts are provided that generate content for the OSCAL project website, supported by the CircleCI ```build``` workflow on the ```master``` branch.
-1. [generate-schema-documentation.sh](generate-schema-documentation.sh): This is used as part of the site generation pipeline to build HTML documentation for the OSCAL models.
+1. [generate-model-documentation.sh](generate-model-documentation.sh): This is used as part of the site generation pipeline to build HTML documentation for the OSCAL models.
 
 # Running the Scripts Locally
 
@@ -177,4 +177,3 @@ jq
 ```
 sudo apt-get install jq
 ```
-