Fixes to build scripts (usnistgov#482)

* Adjusted interface for XSLT file references in JSON converter generation

* Fixed typos in scripts

* ignoring gemerated directories

* Fixed broken link

.gitignore

@@ -15,10 +15,13 @@ backup*
 working/
 __pycache__
 
-# Production outputs
+## Production outputs
 pub/
 target/
 xspec/*.html
+# Bundler
+docs/.bundle
+docs/vendor
 
 # ignore generated content
 /archive

LICENSE.md

@@ -14,7 +14,7 @@ This project includes a mix of the following:
 
 Portions of this software were developed by employees of the National Institute
 of Standards and Technology (NIST), an agency of the Federal Government.
-Pursuant to [title 17 section 105 of the United States Code](https://www.gpo.gov/fdsys/granule/USCODE-2010-title17/USCODE-2010-title17-chap1-sec105), works of NIST employees are
+Pursuant to [title 17 section 105 of the United States Code](https://www.govinfo.gov/content/pkg/USCODE-2010-title17/html/USCODE-2010-title17-chap1-sec105.htm), works of NIST employees are
 not subject to copyright protection in the United States and are considered to
 be in the public domain. Permission to freely use, copy, modify, and distribute
 this software and its documentation without fee is hereby granted, provided that

build/ci-cd/copy-and-convert-content.sh

@@ -15,9 +15,8 @@ HELP=false
 usage() {                                      # Function: Print a help message.
   cat << EOF
 Usage: $0 [options]
-Run all build scripts
 
--h, -help,                        Display help
+-h, --help                        Display help
 -w DIR, --working-dir DIR         Generate artifacts in DIR
 -v                                Provide verbose output
 --keep-temp-scratch-dir           If a scratch directory is automatically
@@ -59,7 +58,7 @@ done
 OTHER_ARGS=$@ # save the remaining args
 
 echo ""
-echo "${P_INFO}Generating XML and JSON Schema${P_END}"
+echo "${P_INFO}Copying and Converting Content${P_END}"
 echo "${P_INFO}==============================${P_END}"
 
 if [ "$VERBOSE" = "true" ]; then
@@ -135,12 +134,15 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
         # Format specific post-processing
         case $altformat in
         json)
+          # Remove extra slashes
+          perl -pi -e 's,\\/,/,g' "${dest}"
+          # translate OSCAL mime types
+          perl -pi -e 's,(application/oscal\.[a-z]+\+)xml\",\1json\",g' "${dest}"
+          # relative content paths
           # translate path names, starting first with the xml directory, then the filename
-          # cat  NIST_SP-800-53_rev4_LOW-baseline_profile.json | perl -lpe 's/(\"(?:(?!xml)[^\/]+\/)*)xml\//\1json/' | perl -lpe 's/(\"(?:[^\/]+\/)*.+(?=\.xml\"))\.xml\"/\1.json\"/'
-          perl -pi -e 's,\\/,/,g' ${dest}
-          perl -pi -e 's,(application/oscal\.[a-z]+\+)xml",\1json",g' ${dest}
-          perl -pi -e 's,/xml/,/json/,g' ${dest}
-          perl -pi -e 's,("(?:[^"/]+/)*[^"]+(?=\.xml"))\.xml",\1.json",g' ${dest}
+          perl -pi -e 's,(\.\./[^\"]+(?=/xml/))/xml/,\1/json/,g' "${dest}"
+          perl -pi -e 's,(\.\./[^\"]+(?=/json/)[^\"]+(?=.xml\")).xml\",\1.json\",g' "${dest}"
+          perl -pi -e 's,(\"[^/\"]+(?=\.xml\")).xml\",\1.json\",g' "${dest}"
 
 #          cp "${dest}.tmp" "${dest}"
 
@@ -168,6 +170,8 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
             exitcode=1
             continue
           fi
+		  # remove carriage returns
+		  perl -pi -e 's,\r,,g' "$dest_pretty"
 
           result=$(validate_json "$schema" "$dest_pretty" 2>&1)
           cmd_exitcode=$?

build/ci-cd/generate-content-converters.sh

@@ -14,13 +14,10 @@ HELP=false
 usage() {                                      # Function: Print a help message.
   cat << EOF
 Usage: $0 [options]
-Run all build scripts
 
--h, -help,                        Display help
+-h, --help                        Display help
 -w DIR, --working-dir DIR         Generate artifacts in DIR
 -v                                Provide verbose output
---keep-temp-scratch-dir           If a scratch directory is automatically
-                                  created, it will not be automatically removed.
 EOF
 }
 
@@ -58,8 +55,8 @@ done
 OTHER_ARGS=$@ # save the remaining args
 
 echo ""
-echo "${P_INFO}Generating XML and JSON Schema${P_END}"
-echo "${P_INFO}==============================${P_END}"
+echo "${P_INFO}Generating XML <-> JSON Content Converters${P_END}"
+echo "${P_INFO}==========================================${P_END}"
 
 if [ "$VERBOSE" = "true" ]; then
   echo "${P_INFO}Using working directory:${P_END} ${WORKING_DIR}"

build/ci-cd/generate-model-documentation.sh

@@ -14,9 +14,8 @@ HELP=false
 usage() {                                      # Function: Print a help message.
   cat << EOF
 Usage: $0 [options]
-Run all build scripts
 
--h, -help,                        Display help
+-h, --help                        Display help
 -w DIR, --working-dir DIR         Generate artifacts in DIR
 -v                                Provide verbose output
 --keep-temp-scratch-dir           If a scratch directory is automatically

build/ci-cd/generate-schema.sh

@@ -15,13 +15,10 @@ HELP=false
 usage() {                                      # Function: Print a help message.
   cat << EOF
 Usage: $0 [options]
-Run all build scripts
 
--h, -help,                        Display help
+-h, --help                        Display help
 -w DIR, --working-dir DIR         Generate artifacts in DIR
 -v                                Provide verbose output
---keep-temp-scratch-dir           If a scratch directory is automatically
-                                  created, it will not be automatically removed.
 EOF
 }
 

build/ci-cd/package-release.sh

@@ -2,7 +2,7 @@
 
 if [[ -z "$OSCALDIR" ]]; then
     DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-    source "$DIR/common-environment.sh"
+    source "$DIR/include/common-environment.sh"
 fi
 
 if [ -z "$1" ]; then
@@ -34,7 +34,7 @@ while IFS="|" read path dest_path || [[ -n "$path" ]]; do
 
   if [[ ! -z "$path" ]]; then
     files_to_process="$OSCALDIR"/"$path"
-    IFS= # disable word splitting    
+    IFS= # disable word splitting
     for file in $files_to_process
     do
       src="$OSCALDIR/$file"

build/ci-cd/run-all.sh

@@ -20,7 +20,7 @@ usage() {                                      # Function: Print a help message.
 Usage: $0 [options]
 Run all build scripts
 
--h, -help,                        Display help
+-h, --help                        Display help
 -w DIR, --working-dir DIR         Generate artifacts in DIR
 -v                                Provide verbose output
 --scratch-dir DIR                 Generate temporary artifacts in DIR

build/ci-cd/validate-content-conversion-round-trips.sh

@@ -9,6 +9,7 @@ fi
 source "$OSCALDIR/build/ci-cd/include/saxon-init.sh"
 
 # Catalog round trip from XML -> JSON -> XML
+MSYS_NO_PATHCONV=1
 
 # Option defaults
 KEEP_TEMP_SCRATCH_DIR=false
@@ -19,9 +20,8 @@ HELP=false
 usage() {                                      # Function: Print a help message.
   cat << EOF
 Usage: $0 [options]
-Run all build scripts
 
--h, -help,                        Display help
+-h, --help                        Display help
 -w DIR, --working-dir DIR         Generate artifacts in DIR
 -v                                Provide verbose output
 --scratch-dir DIR                 Generate temporary artifacts in DIR
@@ -159,14 +159,14 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
 
         # transformation of JSON back to XML
         converter="$WORKING_DIR/xml/convert/oscal_${model}_json-to-xml-converter.xsl"
-        converter_path=$(realpath --relative-to="$PWD" "$converter")
-        back_to_xml_file_relative=$(realpath --relative-to="$PWD" "$back_to_xml_file")
+        converter_path="$(realpath "$converter")"
+        back_to_xml_file_relative="$(realpath --relative-to="$PWD" "$back_to_xml_file")"
 
         # Make the json file relative to the converter
-        converter_dir=$(dirname "$converter_path")
-        json_file_path=$(realpath --relative-to="$converter_dir" "$to_json_file")
+        converter_dir="$(dirname "$converter")"
+        json_file_path="$(realpath "$to_json_file")"
 
-        result=$(xsl_transform "$converter_path" "" "$back_to_xml_file_relative" "-it" "json-file=${json_file_path}" 2>&1)
+        result=$(xsl_transform "$converter_path" "" "$back_to_xml_file" "-it" "json-file=${json_file_path}" 2>&1)
 
         # check the exit code for the conversion
         cmd_exitcode=$?
@@ -183,8 +183,8 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
 
         # Validate the resulting XML
         schema="$WORKING_DIR/xml/schema/oscal_${model}_schema.xsd"
-        schema_relative=$(realpath --relative-to="${WORKING_DIR}" "$schema")
-        result=$(xmllint --noout --schema "$schema" "$back_to_xml_file_relative" 2>&1)
+        schema_relative="$(realpath --relative-to="${WORKING_DIR}" "$schema")"
+        result="$(xmllint --noout --schema "$schema" "$back_to_xml_file" 2>&1)"
         cmd_exitcode=$?
         if [ $cmd_exitcode -ne 0 ]; then
           echo "${P_ERROR}XML Schema validation failed for '${P_END}${back_to_xml_file}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"

build/ci-cd/validate-content.sh

@@ -15,9 +15,8 @@ HELP=false
 usage() {                                      # Function: Print a help message.
   cat << EOF
 Usage: $0 [options]
-Run all build scripts
 
--h, -help,                        Display help
+-h, --help                        Display help
 -w DIR, --working-dir DIR         Generate artifacts in DIR
 -v                                Provide verbose output
 --keep-temp-scratch-dir           If a scratch directory is automatically

build/ci-cd/validate-metaschema.sh

@@ -17,7 +17,7 @@ usage() {                                      # Function: Print a help message.
 Usage: $0 [options]
 Run all build scripts
 
--h, -help,                        Display help
+-h, --help                        Display help
 -w DIR, --working-dir DIR         Generate artifacts in DIR
 -v                                Provide verbose output
 --scratch-dir DIR                 Generate temporary artifacts in DIR
@@ -87,8 +87,8 @@ if [ -z "${SCRATCH_DIR+x}" ]; then
 fi
 
 echo ""
-echo "${P_INFO}Validating Metaschema Defintions${P_END}"
-echo "${P_INFO}================================${P_END}"
+echo "${P_INFO}Validating Metaschema Definitions${P_END}"
+echo "${P_INFO}=================================${P_END}"
 
 if [ "$VERBOSE" = "true" ]; then
   echo "${P_INFO}Using scratch directory:${P_END} ${SCRATCH_DIR}"

build/metaschema/json/produce-json-converter.xsl

@@ -397,10 +397,12 @@
 
         <XSLT:param name="json-file" as="xs:string?"/>
 
-        <XSLT:variable name="json-xml" select="unparsed-text($json-file) ! json-to-xml(.)"/>
+        <XSLT:variable name="using-json-file" select="replace($json-file,'^/','') ! ('file:///' || .)"/>
+
+        <XSLT:variable name="json-xml" select="unparsed-text($using-json-file) ! json-to-xml(.)"/>
 
         <XSLT:template name="xsl:initial-template" match="/">
-                <XSLT:choose>
+            <XSLT:choose>
                 <XSLT:when test="matches($json-file,'\S') and exists($json-xml/map)">
                     <XSLT:apply-templates select="$json-xml" mode="json2xml"/>
                 </XSLT:when>