From dbc7c403f4faf7dc2a7c4776bd063a2cbee65231 Mon Sep 17 00:00:00 2001 From: David Waltermire Date: Fri, 30 Aug 2019 15:15:18 -0400 Subject: [PATCH] Fixes to build scripts (#482) * Adjusted interface for XSLT file references in JSON converter generation * Fixed typos in scripts * ignoring gemerated directories * Fixed broken link --- .gitignore | 5 ++++- LICENSE.md | 2 +- build/ci-cd/copy-and-convert-content.sh | 20 +++++++++++-------- build/ci-cd/generate-content-converters.sh | 9 +++------ build/ci-cd/generate-model-documentation.sh | 3 +-- build/ci-cd/generate-schema.sh | 5 +---- build/ci-cd/package-release.sh | 4 ++-- build/ci-cd/run-all.sh | 2 +- ...validate-content-conversion-round-trips.sh | 18 ++++++++--------- build/ci-cd/validate-content.sh | 3 +-- build/ci-cd/validate-metaschema.sh | 6 +++--- .../json/produce-json-converter.xsl | 6 ++++-- 12 files changed, 42 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 4b1af9f804..c63d185cb7 100644 --- a/.gitignore +++ b/.gitignore @@ -15,10 +15,13 @@ backup* working/ __pycache__ -# Production outputs +## Production outputs pub/ target/ xspec/*.html +# Bundler +docs/.bundle +docs/vendor # ignore generated content /archive diff --git a/LICENSE.md b/LICENSE.md index 09da2f80ea..91145c4f7b 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -14,7 +14,7 @@ This project includes a mix of the following: Portions of this software were developed by employees of the National Institute of Standards and Technology (NIST), an agency of the Federal Government. -Pursuant to [title 17 section 105 of the United States Code](https://www.gpo.gov/fdsys/granule/USCODE-2010-title17/USCODE-2010-title17-chap1-sec105), works of NIST employees are +Pursuant to [title 17 section 105 of the United States Code](https://www.govinfo.gov/content/pkg/USCODE-2010-title17/html/USCODE-2010-title17-chap1-sec105.htm), works of NIST employees are not subject to copyright protection in the United States and are considered to be in the public domain. Permission to freely use, copy, modify, and distribute this software and its documentation without fee is hereby granted, provided that diff --git a/build/ci-cd/copy-and-convert-content.sh b/build/ci-cd/copy-and-convert-content.sh index 4011ba84bb..7e9dd044d9 100755 --- a/build/ci-cd/copy-and-convert-content.sh +++ b/build/ci-cd/copy-and-convert-content.sh @@ -15,9 +15,8 @@ HELP=false usage() { # Function: Print a help message. cat << EOF Usage: $0 [options] -Run all build scripts --h, -help, Display help +-h, --help Display help -w DIR, --working-dir DIR Generate artifacts in DIR -v Provide verbose output --keep-temp-scratch-dir If a scratch directory is automatically @@ -59,7 +58,7 @@ done OTHER_ARGS=$@ # save the remaining args echo "" -echo "${P_INFO}Generating XML and JSON Schema${P_END}" +echo "${P_INFO}Copying and Converting Content${P_END}" echo "${P_INFO}==============================${P_END}" if [ "$VERBOSE" = "true" ]; then @@ -135,12 +134,15 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do # Format specific post-processing case $altformat in json) + # Remove extra slashes + perl -pi -e 's,\\/,/,g' "${dest}" + # translate OSCAL mime types + perl -pi -e 's,(application/oscal\.[a-z]+\+)xml\",\1json\",g' "${dest}" + # relative content paths # translate path names, starting first with the xml directory, then the filename - # cat NIST_SP-800-53_rev4_LOW-baseline_profile.json | perl -lpe 's/(\"(?:(?!xml)[^\/]+\/)*)xml\//\1json/' | perl -lpe 's/(\"(?:[^\/]+\/)*.+(?=\.xml\"))\.xml\"/\1.json\"/' - perl -pi -e 's,\\/,/,g' ${dest} - perl -pi -e 's,(application/oscal\.[a-z]+\+)xml",\1json",g' ${dest} - perl -pi -e 's,/xml/,/json/,g' ${dest} - perl -pi -e 's,("(?:[^"/]+/)*[^"]+(?=\.xml"))\.xml",\1.json",g' ${dest} + perl -pi -e 's,(\.\./[^\"]+(?=/xml/))/xml/,\1/json/,g' "${dest}" + perl -pi -e 's,(\.\./[^\"]+(?=/json/)[^\"]+(?=.xml\")).xml\",\1.json\",g' "${dest}" + perl -pi -e 's,(\"[^/\"]+(?=\.xml\")).xml\",\1.json\",g' "${dest}" # cp "${dest}.tmp" "${dest}" @@ -168,6 +170,8 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do exitcode=1 continue fi + # remove carriage returns + perl -pi -e 's,\r,,g' "$dest_pretty" result=$(validate_json "$schema" "$dest_pretty" 2>&1) cmd_exitcode=$? diff --git a/build/ci-cd/generate-content-converters.sh b/build/ci-cd/generate-content-converters.sh index 69c062f229..3e6e975bd6 100755 --- a/build/ci-cd/generate-content-converters.sh +++ b/build/ci-cd/generate-content-converters.sh @@ -14,13 +14,10 @@ HELP=false usage() { # Function: Print a help message. cat << EOF Usage: $0 [options] -Run all build scripts --h, -help, Display help +-h, --help Display help -w DIR, --working-dir DIR Generate artifacts in DIR -v Provide verbose output ---keep-temp-scratch-dir If a scratch directory is automatically - created, it will not be automatically removed. EOF } @@ -58,8 +55,8 @@ done OTHER_ARGS=$@ # save the remaining args echo "" -echo "${P_INFO}Generating XML and JSON Schema${P_END}" -echo "${P_INFO}==============================${P_END}" +echo "${P_INFO}Generating XML <-> JSON Content Converters${P_END}" +echo "${P_INFO}==========================================${P_END}" if [ "$VERBOSE" = "true" ]; then echo "${P_INFO}Using working directory:${P_END} ${WORKING_DIR}" diff --git a/build/ci-cd/generate-model-documentation.sh b/build/ci-cd/generate-model-documentation.sh index c97b6036e4..e3ca192a9c 100755 --- a/build/ci-cd/generate-model-documentation.sh +++ b/build/ci-cd/generate-model-documentation.sh @@ -14,9 +14,8 @@ HELP=false usage() { # Function: Print a help message. cat << EOF Usage: $0 [options] -Run all build scripts --h, -help, Display help +-h, --help Display help -w DIR, --working-dir DIR Generate artifacts in DIR -v Provide verbose output --keep-temp-scratch-dir If a scratch directory is automatically diff --git a/build/ci-cd/generate-schema.sh b/build/ci-cd/generate-schema.sh index defe13fb7a..dddd1c5fd1 100755 --- a/build/ci-cd/generate-schema.sh +++ b/build/ci-cd/generate-schema.sh @@ -15,13 +15,10 @@ HELP=false usage() { # Function: Print a help message. cat << EOF Usage: $0 [options] -Run all build scripts --h, -help, Display help +-h, --help Display help -w DIR, --working-dir DIR Generate artifacts in DIR -v Provide verbose output ---keep-temp-scratch-dir If a scratch directory is automatically - created, it will not be automatically removed. EOF } diff --git a/build/ci-cd/package-release.sh b/build/ci-cd/package-release.sh index 0c352465ae..2ab5ebdcd1 100755 --- a/build/ci-cd/package-release.sh +++ b/build/ci-cd/package-release.sh @@ -2,7 +2,7 @@ if [[ -z "$OSCALDIR" ]]; then DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - source "$DIR/common-environment.sh" + source "$DIR/include/common-environment.sh" fi if [ -z "$1" ]; then @@ -34,7 +34,7 @@ while IFS="|" read path dest_path || [[ -n "$path" ]]; do if [[ ! -z "$path" ]]; then files_to_process="$OSCALDIR"/"$path" - IFS= # disable word splitting + IFS= # disable word splitting for file in $files_to_process do src="$OSCALDIR/$file" diff --git a/build/ci-cd/run-all.sh b/build/ci-cd/run-all.sh index 2c1db8107c..a00f2682b2 100755 --- a/build/ci-cd/run-all.sh +++ b/build/ci-cd/run-all.sh @@ -20,7 +20,7 @@ usage() { # Function: Print a help message. Usage: $0 [options] Run all build scripts --h, -help, Display help +-h, --help Display help -w DIR, --working-dir DIR Generate artifacts in DIR -v Provide verbose output --scratch-dir DIR Generate temporary artifacts in DIR diff --git a/build/ci-cd/validate-content-conversion-round-trips.sh b/build/ci-cd/validate-content-conversion-round-trips.sh index 9129909a03..2b55fc32c6 100755 --- a/build/ci-cd/validate-content-conversion-round-trips.sh +++ b/build/ci-cd/validate-content-conversion-round-trips.sh @@ -9,6 +9,7 @@ fi source "$OSCALDIR/build/ci-cd/include/saxon-init.sh" # Catalog round trip from XML -> JSON -> XML +MSYS_NO_PATHCONV=1 # Option defaults KEEP_TEMP_SCRATCH_DIR=false @@ -19,9 +20,8 @@ HELP=false usage() { # Function: Print a help message. cat << EOF Usage: $0 [options] -Run all build scripts --h, -help, Display help +-h, --help Display help -w DIR, --working-dir DIR Generate artifacts in DIR -v Provide verbose output --scratch-dir DIR Generate temporary artifacts in DIR @@ -159,14 +159,14 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do # transformation of JSON back to XML converter="$WORKING_DIR/xml/convert/oscal_${model}_json-to-xml-converter.xsl" - converter_path=$(realpath --relative-to="$PWD" "$converter") - back_to_xml_file_relative=$(realpath --relative-to="$PWD" "$back_to_xml_file") + converter_path="$(realpath "$converter")" + back_to_xml_file_relative="$(realpath --relative-to="$PWD" "$back_to_xml_file")" # Make the json file relative to the converter - converter_dir=$(dirname "$converter_path") - json_file_path=$(realpath --relative-to="$converter_dir" "$to_json_file") + converter_dir="$(dirname "$converter")" + json_file_path="$(realpath "$to_json_file")" - result=$(xsl_transform "$converter_path" "" "$back_to_xml_file_relative" "-it" "json-file=${json_file_path}" 2>&1) + result=$(xsl_transform "$converter_path" "" "$back_to_xml_file" "-it" "json-file=${json_file_path}" 2>&1) # check the exit code for the conversion cmd_exitcode=$? @@ -183,8 +183,8 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do # Validate the resulting XML schema="$WORKING_DIR/xml/schema/oscal_${model}_schema.xsd" - schema_relative=$(realpath --relative-to="${WORKING_DIR}" "$schema") - result=$(xmllint --noout --schema "$schema" "$back_to_xml_file_relative" 2>&1) + schema_relative="$(realpath --relative-to="${WORKING_DIR}" "$schema")" + result="$(xmllint --noout --schema "$schema" "$back_to_xml_file" 2>&1)" cmd_exitcode=$? if [ $cmd_exitcode -ne 0 ]; then echo "${P_ERROR}XML Schema validation failed for '${P_END}${back_to_xml_file}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}" diff --git a/build/ci-cd/validate-content.sh b/build/ci-cd/validate-content.sh index 8be4aa4c45..538abc9c06 100755 --- a/build/ci-cd/validate-content.sh +++ b/build/ci-cd/validate-content.sh @@ -15,9 +15,8 @@ HELP=false usage() { # Function: Print a help message. cat << EOF Usage: $0 [options] -Run all build scripts --h, -help, Display help +-h, --help Display help -w DIR, --working-dir DIR Generate artifacts in DIR -v Provide verbose output --keep-temp-scratch-dir If a scratch directory is automatically diff --git a/build/ci-cd/validate-metaschema.sh b/build/ci-cd/validate-metaschema.sh index e926d43cfa..da52cc4a44 100755 --- a/build/ci-cd/validate-metaschema.sh +++ b/build/ci-cd/validate-metaschema.sh @@ -17,7 +17,7 @@ usage() { # Function: Print a help message. Usage: $0 [options] Run all build scripts --h, -help, Display help +-h, --help Display help -w DIR, --working-dir DIR Generate artifacts in DIR -v Provide verbose output --scratch-dir DIR Generate temporary artifacts in DIR @@ -87,8 +87,8 @@ if [ -z "${SCRATCH_DIR+x}" ]; then fi echo "" -echo "${P_INFO}Validating Metaschema Defintions${P_END}" -echo "${P_INFO}================================${P_END}" +echo "${P_INFO}Validating Metaschema Definitions${P_END}" +echo "${P_INFO}=================================${P_END}" if [ "$VERBOSE" = "true" ]; then echo "${P_INFO}Using scratch directory:${P_END} ${SCRATCH_DIR}" diff --git a/build/metaschema/json/produce-json-converter.xsl b/build/metaschema/json/produce-json-converter.xsl index 87da416d68..03e0cd3d7c 100644 --- a/build/metaschema/json/produce-json-converter.xsl +++ b/build/metaschema/json/produce-json-converter.xsl @@ -397,10 +397,12 @@ - + + + - +