Merge pull request #287 from CDCgov/dev_file_system_report_schema_loader

Dev file system report schema loader

libs/schema-validation/src/main/kotlin/gov/cdc/ocio/reportschemavalidator/loaders/FileSchemaLoader.kt

@@ -1,23 +1,29 @@
 package gov.cdc.ocio.reportschemavalidator.loaders
 
 import gov.cdc.ocio.reportschemavalidator.models.SchemaFile
+import java.io.FileNotFoundException
 
 
 /**
  * The class which loads the schema files from the class path
  */
-class FileSchemaLoader : SchemaLoader {
+class FileSchemaLoader(private val config: Map<String, String>) : SchemaLoader {
 
     /**
      * The function which loads the schema based on the file name path and returns a [SchemaFile]
      * @param fileName String
      * @return [SchemaFile]
      */
     override fun loadSchemaFile(fileName: String): SchemaFile {
-        val schemaDirectoryPath = "schema"
+        val schemaLocalSystemFilePath = config["REPORT_SCHEMA_LOCAL_FILE_SYSTEM_PATH"]
+            ?: throw IllegalArgumentException("Local file system path is not configured")
+        val file = java.io.File("$schemaLocalSystemFilePath/$fileName")
+        if (!file.exists()) {
+            throw FileNotFoundException("Report rejected: file - ${fileName} not found for content schema.")
+        }
         return SchemaFile(
             fileName = fileName,
-            inputStream = javaClass.classLoader.getResourceAsStream("$schemaDirectoryPath/$fileName")
+            inputStream = file.inputStream()
         )
     }
 

libs/schema-validation/src/main/kotlin/gov/cdc/ocio/reportschemavalidator/service/SchemaValidationService.kt

@@ -11,6 +11,7 @@ import gov.cdc.ocio.reportschemavalidator.models.SchemaFile
 import gov.cdc.ocio.reportschemavalidator.validators.SchemaValidator
 import mu.KLogger
 import gov.cdc.ocio.reportschemavalidator.utils.JsonUtils
+import java.io.FileNotFoundException
 
 
 /**
@@ -84,14 +85,11 @@ class SchemaValidationService(
         }
         catch (e: MalformedException){
             val reason = "Report rejected: Malformed JSON or error processing the report"
-            e.message?.let { invalidData.add(it) }
-            //TODO :This should be done in the calling code
-            // val malformedReportMessage = safeParseMessageAsReport(message)
-            return errorProcessor.processError(
-                reason,
-                schemaFileNames,
-                invalidData
-            )
+            return processValidationErrors(reason, invalidData, schemaFileNames)
+        }
+        catch (e: FileNotFoundException){
+            val reason =  e.message ?: "Report rejected: Content schema file not found"
+            return processValidationErrors(reason, invalidData, schemaFileNames)
         }
         return ValidationSchemaResult(
             "Successfully validated the report schema",
@@ -294,4 +292,21 @@ class SchemaValidationService(
     private fun getContentSchemaVersionNode(contentNode: JsonNode): JsonNode? {
         return contentNode.get("content_schema_version")
     }
+
+    /**
+     * The function to process the Malformed and FileNotFound exception errors
+     * @param reason String
+     * @param invalidData MutableList<String>
+     * @param schemaFileNames MutableList<String>
+     * @return ValidationSchemaResult
+     */
+
+    private fun processValidationErrors(reason:String, invalidData: MutableList<String>, schemaFileNames: MutableList<String>):ValidationSchemaResult{
+        reason.let { invalidData.add(it) }
+        return errorProcessor.processError(
+            reason,
+            schemaFileNames,
+            invalidData
+        )
+    }
 }

libs/schema-validation/src/main/kotlin/gov/cdc/ocio/reportschemavalidator/utils/FileSystemConfiguration.kt

@@ -0,0 +1,14 @@
+package gov.cdc.ocio.reportschemavalidator.utils
+
+import io.ktor.server.config.*
+
+/**
+ * Blob storage configuration class
+ * @param config ApplicationConfig
+ * @param configurationPath String?
+ */
+class FileSystemConfiguration(config: ApplicationConfig, configurationPath: String? = null) {
+    private val configPath = if (configurationPath != null) "$configurationPath." else ""
+    val localFileSystemPath = config.tryGetString("${configPath}.report_schema_local_path") ?: ""
+
+}

libs/schema-validation/src/main/kotlin/gov/cdc/ocio/reportschemavalidator/utils/CloudSchemaLoaderConfiguration.kt → libs/schema-validation/src/main/kotlin/gov/cdc/ocio/reportschemavalidator/utils/SchemaLoaderConfiguration.kt

@@ -2,25 +2,27 @@ package gov.cdc.ocio.reportschemavalidator.utils
 
 
 import gov.cdc.ocio.reportschemavalidator.loaders.CloudSchemaLoader
+import gov.cdc.ocio.reportschemavalidator.loaders.FileSchemaLoader
+import gov.cdc.ocio.reportschemavalidator.loaders.SchemaLoader
 import io.ktor.server.application.*
 import io.ktor.server.config.*
 
 /**
  * The class which is used to create the schema loader instance based on env vars
  * @param environment ApplicationEnvironment
  */
-class CloudSchemaLoaderConfiguration(environment: ApplicationEnvironment){
-    private val schemaLoaderSystem = environment.config.tryGetString("ktor.schema_loader_system")?: ""
+class SchemaLoaderConfiguration(environment: ApplicationEnvironment){
+    private val schemaLoaderSystem = environment.config.tryGetString("ktor.report_schema_loader_system")?: ""
     private val s3Bucket = environment.config.tryGetString("aws.s3.report_schema_bucket") ?: ""
     private val s3Region = environment.config.tryGetString("aws.s3.report_schema_region") ?: ""
     private val connectionString = environment.config.tryGetString("azure.blob_storage.report_schema_connection_string") ?: ""
     private val container = environment.config.tryGetString("azure.blob_storage.report_schema_container") ?: ""
-
+    private val localFileSystemPath = environment.config.tryGetString("file_system.report_schema_local_path") ?: ""
     /**
      * The function which instantiates the CloudSchemaLoader based on the schema loader system type
      * @return CloudSchemaLoader
      */
-    fun createSchemaLoader(): CloudSchemaLoader {
+    fun createSchemaLoader(): SchemaLoader {
         when (schemaLoaderSystem.lowercase()) {
             SchemaLoaderSystemType.S3.toString().lowercase()  -> {
                 val config = mapOf(
@@ -37,6 +39,14 @@ class CloudSchemaLoaderConfiguration(environment: ApplicationEnvironment){
                 )
                 return CloudSchemaLoader(schemaLoaderSystem, config)
             }
+
+            SchemaLoaderSystemType.FILE_SYSTEM.toString().lowercase() -> {
+                val config = mapOf(
+                    "REPORT_SCHEMA_LOCAL_FILE_SYSTEM_PATH" to localFileSystemPath
+
+                )
+                return FileSchemaLoader(config)
+            }
             else ->throw IllegalArgumentException( "Unsupported schema loader type: $schemaLoaderSystem")
 
         }

libs/schema-validation/src/main/kotlin/gov/cdc/ocio/reportschemavalidator/utils/CloudSchemaLoaderConfigurationKoinCreator.kt → libs/schema-validation/src/main/kotlin/gov/cdc/ocio/reportschemavalidator/utils/SchemaLoaderConfigurationKoinCreator.kt

@@ -8,10 +8,11 @@ import org.koin.dsl.module
 /**
  * Helper class for creating koin modules for a report schema loader.
  */
-class CloudSchemaLoaderConfigurationKoinCreator {
+class SchemaLoaderConfigurationKoinCreator {
 
     companion object {
 
+
         /**
          * The class which loads the specific cloud schema loader configuration based on the env vars
          * @param environment ApplicationEnvironment
@@ -21,7 +22,7 @@ class CloudSchemaLoaderConfigurationKoinCreator {
             val logger = KotlinLogging.logger {}
 
             val schemaLoaderSystemModule = module {
-                val schemaLoaderSystem = environment.config.property("ktor.schema_loader_system").getString()
+                val schemaLoaderSystem = environment.config.property("ktor.report_schema_loader_system").getString()
                 val schemaLoaderSystemType: SchemaLoaderSystemType
                 when (schemaLoaderSystem.lowercase()) {
                     SchemaLoaderSystemType.S3.toString().lowercase() -> {
@@ -34,6 +35,11 @@ class CloudSchemaLoaderConfigurationKoinCreator {
                         schemaLoaderSystemType = SchemaLoaderSystemType.BLOB_STORAGE
                     }
 
+                    SchemaLoaderSystemType.FILE_SYSTEM.toString().lowercase() -> {
+                        single {  FileSystemConfiguration(environment.config,configurationPath = "file_system") }
+                        schemaLoaderSystemType = SchemaLoaderSystemType.FILE_SYSTEM
+                    }
+
                     else -> {
                         val msg = "Unsupported schema loader type: $schemaLoaderSystem"
                         logger.error { msg }

libs/schema-validation/src/main/kotlin/gov/cdc/ocio/reportschemavalidator/utils/SchemaLoaderSystemType.kt

@@ -3,5 +3,5 @@ package gov.cdc.ocio.reportschemavalidator.utils
 enum class SchemaLoaderSystemType {
     S3,
     BLOB_STORAGE,
-    UNKNOWN
+    FILE_SYSTEM
 }

libs/schema-validation/src/test/kotlin/test/ReportSchemaValidationTests.kt

@@ -20,6 +20,8 @@ import org.testng.Assert
 import org.testng.annotations.BeforeMethod
 import org.testng.annotations.Test
 import java.io.File
+import java.nio.file.Paths
+
 
 class ReportSchemaValidationTests {
 
@@ -32,8 +34,6 @@ class ReportSchemaValidationTests {
     private val schemaValidator: SchemaValidator = JsonSchemaValidator(logger)
     //  Mock the jsonUtils dependency
     private val jsonUtils: JsonUtils = DefaultJsonUtils(objectMapper)
-    // Mock the schemaValidator dependency
-    private val schemaLoader: SchemaLoader = FileSchemaLoader()
     //Base validation failure reason
     private var reason = "The report could not be validated against the JSON schema: base.1.0.0.schema.json."
 
@@ -43,6 +43,25 @@ class ReportSchemaValidationTests {
     fun setUp() {
         MockitoAnnotations.openMocks(this)
 
+        // Current directory: C:\apps\dex\data-exchange-processing-status\libs\schema-validation
+        val currentPath = Paths.get("..").toAbsolutePath().normalize()
+
+        // Navigate to the "reports" directory
+        val reportsPath = currentPath.resolveSibling("reports").normalize()
+
+        val path = Paths.get(reportsPath.toString())
+        val absolutePath = path.toFile()
+        if (absolutePath.exists()) {
+            println("Directory exists")
+        } else {
+            println("Directory does not exist")
+        }
+
+        val schemaLoader:SchemaLoader = FileSchemaLoader(mapOf(
+            "REPORT_SCHEMA_LOCAL_FILE_SYSTEM_PATH" to absolutePath.toString()
+        ))
+
+
         schemaValidationService = SchemaValidationService(
             schemaLoader,
             schemaValidator,
@@ -146,7 +165,8 @@ class ReportSchemaValidationTests {
         val testMessage =File("./src/test/kotlin/data/report_schema_contentSchemaVersion_validation.json").readBytes()
         val message = createMessageFromBinary(testMessage)
         val result: ValidationSchemaResult = schemaValidationService.validateJsonSchema(message)
-        val missingContent ="Report rejected: Content schema file not found for content schema name 'hl7v2-debatch' and schema version '2.0.0'."
+        val missingContent ="Report rejected: file - hl7v2-debatch.2.0.0.schema.json not found for content schema."
+        //"Report rejected: file: ${file.absolutePath} not found for content schema"
         Assert.assertTrue(!result.status)
         Assert.assertEquals(result.reason,missingContent)
         Assert.assertNotSame(result.invalidData, mutableListOf<String>())

pstatus-graphql-ktor/README.md

@@ -68,7 +68,18 @@ For Couchbase DB only, set the following environment variables:
 For Mongo DB only, set the following environment variables:
  -`MONGO_CONNECTION_STRING` - URI of the couchbase database.
  - `MONGO_DATABASE_NAME` - Name of the database. For example, "ProcessingStatus".
- - 
+
+For report schema loader, set the following environment variables:
+- `REPORT_SCHEMA_LOADER_SYSTEM` - One of these values (s3, blob_storage or file_system)
+- For `s3` -
+- `REPORT_SCHEMA_S3_BUCKET` - S3 Bucket Name
+- `REPORT_SCHEMA_S3_REGION` - S3 Region.
+- For `blob_storage`
+- `REPORT_SCHEMA_BLOB_CONNECTION_STR` - Connection string of the storage account.
+- `REPORT_SCHEMA_BLOB_CONTAINER` - Blob container name.
+-  For `file_system`
+- `REPORT_SCHEMA_LOCAL_FILE_SYSTEM_PATH`- The local file system path where the reports schema reside
+
 ### GRAPHQL 
 - `GRAPHQL_PATH` - The path of the `GraphQL endpoint`.
 

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/Application.kt

@@ -3,7 +3,7 @@ package gov.cdc.ocio.processingstatusapi
 import gov.cdc.ocio.database.utils.DatabaseKoinCreator
 import gov.cdc.ocio.processingstatusapi.plugins.configureRouting
 import gov.cdc.ocio.processingstatusapi.plugins.graphQLModule
-import gov.cdc.ocio.reportschemavalidator.utils.CloudSchemaLoaderConfigurationKoinCreator
+import gov.cdc.ocio.reportschemavalidator.utils.SchemaLoaderConfigurationKoinCreator
 import graphql.scalars.ExtendedScalars
 import graphql.schema.idl.RuntimeWiring
 import io.ktor.serialization.jackson.*
@@ -18,8 +18,9 @@ import org.koin.ktor.plugin.Koin
 fun KoinApplication.loadKoinModules(environment: ApplicationEnvironment): KoinApplication {
     val databaseModule = DatabaseKoinCreator.moduleFromAppEnv(environment)
     val healthCheckDatabaseModule = DatabaseKoinCreator.dbHealthCheckModuleFromAppEnv(environment)
-    val cloudSchemaConfigurationModule = CloudSchemaLoaderConfigurationKoinCreator.getSchemaLoaderConfigurationFromAppEnv(environment)
-    return modules(listOf(databaseModule, healthCheckDatabaseModule,cloudSchemaConfigurationModule))
+    val schemaConfigurationModule = SchemaLoaderConfigurationKoinCreator.getSchemaLoaderConfigurationFromAppEnv(environment)
+    return modules(listOf(databaseModule, healthCheckDatabaseModule,schemaConfigurationModule))
+
 }
 
 fun main(args: Array<String>) {
@@ -40,4 +41,4 @@ fun Application.module() {
 
     // See https://opensource.expediagroup.com/graphql-kotlin/docs/schema-generator/writing-schemas/scalars
     RuntimeWiring.newRuntimeWiring().scalar(ExtendedScalars.Date)
-}
+}

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/mutations/ReportMutation.kt

@@ -43,15 +43,15 @@ class ReportMutation(private val environment: ApplicationEnvironment) : Mutation
         "*Action*: Can be one of the following values\n"
                 + "`create`: Creates a new report.\n"
                 + "`replace`: Replace an existing report.\n"
-        )
-        action: String,
-
-        @GraphQLDescription(
-            "*Report* to be created or updated, which is the JSON of the report provided.\n"
-        )
-        report: BasicHashMap<String, Any?>
-        ) = run {
+    )
+                      action: String,
+                      @GraphQLDescription(
+                          "*Report* to be created or updated, which is the JSON of the report provided.\n"
+                      )
+                      report: BasicHashMap<String, Any?>
+    ) = run {
         val service = ReportMutationService(environment)
         service.upsertReport(action, report)
     }
 }
+

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/services/ReportMutationService.kt

@@ -18,9 +18,9 @@ import gov.cdc.ocio.processingstatusapi.mutations.models.ValidatedReportResult
 import gov.cdc.ocio.processingstatusapi.services.ValidationComponents.gson
 import gov.cdc.ocio.reportschemavalidator.errors.ErrorLoggerProcessor
 import gov.cdc.ocio.reportschemavalidator.exceptions.ValidationException
-import gov.cdc.ocio.reportschemavalidator.loaders.CloudSchemaLoader
+import gov.cdc.ocio.reportschemavalidator.loaders.SchemaLoader
 import gov.cdc.ocio.reportschemavalidator.service.SchemaValidationService
-import gov.cdc.ocio.reportschemavalidator.utils.CloudSchemaLoaderConfiguration
+import gov.cdc.ocio.reportschemavalidator.utils.SchemaLoaderConfiguration
 import gov.cdc.ocio.reportschemavalidator.utils.DefaultJsonUtils
 import gov.cdc.ocio.reportschemavalidator.validators.JsonSchemaValidator
 import io.ktor.server.application.*
@@ -98,7 +98,9 @@ class ReportMutationService(private val environment: ApplicationEnvironment){
             val actionType = validateAction(action)
 
             //schema loader
-            val schemaLoader = CloudSchemaLoaderConfiguration(environment).createSchemaLoader()
+
+            val schemaLoader = SchemaLoaderConfiguration(environment).createSchemaLoader()
+
 
             // Validate the report
             val validationResult = validateReport(schemaLoader,mapOfContent)
@@ -150,7 +152,9 @@ class ReportMutationService(private val environment: ApplicationEnvironment){
      * @throws Exception
      */
     @Throws(ContentException::class, Exception::class)
-    private fun validateReport(schemaLoader: CloudSchemaLoader, input: Map<String, Any?>?): ValidatedReportResult {
+
+    private fun validateReport(schemaLoader: SchemaLoader, input: Map<String, Any?>?): ValidatedReportResult {
+
         if (input.isNullOrEmpty()) throw ContentException("Can't validate a null or empty report")
 
         try {
@@ -237,4 +241,4 @@ class ReportMutationService(private val environment: ApplicationEnvironment){
         }
 
     }
-}
+}

pstatus-graphql-ktor/src/main/resources/application.conf

@@ -10,7 +10,8 @@ ktor {
 
     version = "0.0.5"
     database = ${DATABASE}
-    schema_loader_system= ${SCHEMA_LOADER_SYSTEM}
+    report_schema_loader_system = ${REPORT_SCHEMA_LOADER_SYSTEM}
+
 }
 
 graphql {
@@ -69,3 +70,7 @@ couchbase {
     username = ${?COUCHBASE_USERNAME}
     password = ${?COUCHBASE_PASSWORD}
 }
+
+file_system{
+    report_schema_local_path= ${REPORT_SCHEMA_LOCAL_FILE_SYSTEM_PATH}
+}

pstatus-report-sink-ktor/README.md

@@ -33,6 +33,17 @@ For Mongo DB only, set the following environment variables:
 - `MONGO_CONNECTION_STRING` - URI of the couchbase database.
 - `MONGO_DATABASE_NAME` - Name of the database. For example, "ProcessingStatus".
 
+For report schema loader, set the following environment variables:
+- `REPORT_SCHEMA_LOADER_SYSTEM` - One of these values (s3, blob_storage or file_system)
+- For `s3` -
+- `REPORT_SCHEMA_S3_BUCKET` - S3 Bucket Name
+- `REPORT_SCHEMA_S3_REGION` - S3 Region.
+- For `blob_storage`
+- `REPORT_SCHEMA_BLOB_CONNECTION_STR` - Connection string of the storage account.
+- `REPORT_SCHEMA_BLOB_CONTAINER` - Blob container name.
+-  For `file_system`
+- `REPORT_SCHEMA_LOCAL_FILE_SYSTEM_PATH`- The local file system path where the reports schema reside
+
 #### Message System
 - The `MSG_SYSTEM` environment variable is used to determine which system will be loaded dynamically.
 Set this variable to one of the following values: