Merge pull request #557 from boozallen/548-record-relation-validation

#548 record with relations should be able to validate reference record fields

foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/java/JavaRecord.java

@@ -156,10 +156,7 @@ private void addSimpleTypeImport(JavaRecordField field) {
     private void addRelationImports() {
         for (Relation relation: wrapped.getRelations()) {
             JavaRecordRelation relationDecorator = new JavaRecordRelation(relation);
-            String classImport = relationDecorator.getGeneratedClassImport();
-            if (classImport !=null) {
-                imports.add(classImport);
-            }
+            imports.addAll(relationDecorator.getGeneratedClassImport());
         }
     }
 

foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/java/JavaRecordRelation.java

@@ -13,6 +13,12 @@
 import com.boozallen.aiops.mda.metamodel.element.BaseRecordRelationDecorator;
 import com.boozallen.aiops.mda.metamodel.element.Relation;
 
+import java.util.HashSet;
+import java.util.Set;
+
+import static com.boozallen.aiops.mda.metamodel.element.util.JavaElementUtils.LIST_IMPORT;
+import static com.boozallen.aiops.mda.metamodel.element.util.JavaElementUtils.VALIDATION_EXCEPTION_IMPORT;
+
 /**
  * Decorates RecordRelation with Java-specific functionality.
  */
@@ -39,10 +45,14 @@ public String getLowercaseName() {
      * Returns the import for the generating the setters/getters of the reference record.
      * @return generated class import
      */
-    public String getGeneratedClassImport() {
+    public Set<String> getGeneratedClassImport() {
+        Set<String> relationImports = new HashSet();
         if (isOneToManyRelation())
-           return "java.util.List";
-        return null;
+           relationImports.add(LIST_IMPORT);
+        if (!this.isNullable()) {
+            relationImports.add(VALIDATION_EXCEPTION_IMPORT);
+        }
+        return relationImports;
     }
 
     /**
@@ -80,4 +90,45 @@ public String getRelationPropDeclaration() {
         return String.format("private %s %s = null;", type, getLowercaseName());
     }
 
+    /**
+     * Returns the reference record validation logic for generating the base record class
+     * @return the reference record validation logic
+     */
+    public String getRelationValidate() {
+        if (this.isOneToManyRelation()) {
+            String validate = """
+                    for (%s record : this.%s) {
+                        record.validate();
+                    }""";
+            return String.format(validate, getName(), getLowercaseName());
+        } else {
+            return String.format("this.%s.validate();",getLowercaseName());
+        }
+    }
+
+    /**
+     * Returns the required reference record validation logic for generating the base record class
+     * @return the required reference record validation logic
+     */
+    public String getRequiredRelationValidate() {
+        String lowercaseName = getLowercaseName();
+        if (this.isOneToManyRelation()) {
+            String validate = """
+                    if (this.%s == null || this.%s.size() == 0) {
+                        throw new ValidationException("Relation record '%s' is required");
+                    } else {
+                        %s
+                    }""";
+            return String.format(validate, lowercaseName, lowercaseName, getName(), getRelationValidate());
+        } else {
+            String validate = """
+                    if (this.%s == null) {
+                        throw new ValidationException("Relation record '%s' is required");
+                    } else {
+                        %s
+                    }""";
+            return String.format(validate, lowercaseName, getName(), getRelationValidate());
+        }
+    }
+
 }

foundation/foundation-mda/src/main/java/com/boozallen/aiops/mda/metamodel/element/python/PythonRecordRelation.java

@@ -95,4 +95,43 @@ public String getRelationPropDeclaration() {
         }
     }
 
+    /**
+     * Returns the reference record validation logic for generating the base record class
+     * @return the reference record validation logic
+     */
+    public String getRelationValidate() {
+        String snakeCaseName = getSnakeCaseName();
+        if (this.isOneToManyRelation()) {
+            String validate = """
+                    for %s in self._%s:
+                                    %s.validate()""";
+            return String.format(validate, snakeCaseName, snakeCaseName, snakeCaseName);
+        } else {
+            return String.format("self._%s.validate()", snakeCaseName);
+        }
+    }
+
+    /**
+     * Returns the required reference record validation logic for generating the base record class
+     * @return the required reference record validation logic
+     */
+    public String getRequiredRelationValidate() {
+        String snakeCaseName = getSnakeCaseName();
+        if (this.isOneToManyRelation()) {
+            String validate = """
+                    if self._%s is None or len(self._%s) == 0:
+                                raise ValueError('Relation record "%s" is required')
+                            else:
+                                %s""";
+            return String.format(validate, snakeCaseName, snakeCaseName, getName(), getRelationValidate());
+        } else {
+            String validate = """
+                    if self._%s is None:
+                                raise ValueError('Relation record "%s" is required')
+                            else:
+                                %s""";
+            return String.format(validate, snakeCaseName, getName(), getRelationValidate());
+        }
+    }
+
 }

foundation/foundation-mda/src/main/resources/templates/data-delivery-data-records/record.base.java.vm

@@ -65,6 +65,9 @@ public abstract class ${record.capitalizedName}Base {
     #if ($record.hasFieldValidations())
         validateFields();
     #end
+    #if ($record.hasRelations())
+         validateRelations();
+    #end
     }
 #if ($record.hasFieldValidations())
 
@@ -93,6 +96,23 @@ public abstract class ${record.capitalizedName}Base {
     }
 #end
 
+#if ($record.hasRelations())
+    /**
+     * Validate the reference records fields.
+     */
+    private void validateRelations() {
+    #foreach ($relation in $record.relations)
+        #if (!$relation.isNullable())
+        ${relation.requiredRelationValidate}
+        #else
+        if (this.${relation.lowercaseName} != null) {
+            ${relation.relationValidate}
+        }
+        #end
+    #end
+    }
+#end
+
     /**
      * Returns the value of the given field for this record.
      * 

foundation/foundation-mda/src/main/resources/templates/data-delivery-data-records/record.base.py.vm

@@ -94,9 +94,11 @@ class ${record.capitalizedName}Base(ABC):
         #end
         #foreach ($relation in $record.relations)
             #if ($relation.isOneToManyRelation())
-            record.${relation.snakeCaseName} = [${relation.name}.from_dict(${relation.snakeCaseName}) for ${relation.snakeCaseName} in dict_obj.get('${relation.name}')]
+            if dict_obj.get('${relation.name}') is not None:
+                record.${relation.snakeCaseName} = [${relation.name}.from_dict(${relation.snakeCaseName}) for ${relation.snakeCaseName} in dict_obj.get('${relation.name}')]
             #else
-            record.${relation.snakeCaseName} = ${relation.name}.from_dict(dict_obj.get('${relation.name}'))
+            if dict_obj.get('${relation.name}') is not None:
+                record.${relation.snakeCaseName} = ${relation.name}.from_dict(dict_obj.get('${relation.name}'))
             #end
         #end
             return record
@@ -123,9 +125,11 @@ class ${record.capitalizedName}Base(ABC):
     #end
     #foreach ($relation in $record.relations)
         #if ($relation.isOneToManyRelation())
-        dict_obj['${relation.name}'] = [${relation.snakeCaseName}.as_dict() for ${relation.snakeCaseName} in self.${relation.snakeCaseName}]
+        if self.${relation.snakeCaseName} is not None and len(self.${relation.snakeCaseName}) > 0:
+            dict_obj['${relation.name}'] = [${relation.snakeCaseName}.as_dict() for ${relation.snakeCaseName} in self.${relation.snakeCaseName}]
         #else
-        dict_obj['${relation.name}'] = self.${relation.snakeCaseName}.as_dict()
+        if self.${relation.snakeCaseName} is not None:
+            dict_obj['${relation.name}'] = self.${relation.snakeCaseName}.as_dict()
         #end
     #end
         return dict_obj
@@ -177,9 +181,15 @@ class ${record.capitalizedName}Base(ABC):
         """
         Performs the validation for this record.
         """
-    #if ($record.hasFieldValidations())
+    #set($hasFieldValidations = $record.hasFieldValidations())
+    #set($hasRelations = $record.hasRelations())
+    #if ($hasFieldValidations)
         self.validate_fields()
-    #else
+    #end
+    #if ($hasRelations)
+        self.validate_relations()
+    #end
+    #if (!$hasFieldValidations && !$hasRelations)
         pass
     #end
 
@@ -203,6 +213,21 @@ class ${record.capitalizedName}Base(ABC):
     #end
 #end
 
+#if ($record.hasRelations())
+    def validate_relations(self) -> None:
+        """
+        Validate the reference records fields.
+        """
+    #foreach ($relation in $record.relations)
+        #if (!$relation.isNullable())
+        ${relation.requiredRelationValidate}
+        #else
+        if self._${relation.snakeCaseName} is not None:
+            ${relation.relationValidate}
+        #end
+    #end
+#end
+
     def get_value_by_field(self, field: ${record.capitalizedName}Field) -> any:
         """
         Returns the value of the given field for this record.

foundation/foundation-mda/src/main/resources/templates/data-delivery-data-records/spark.schema.base.java.vm

@@ -152,10 +152,11 @@ public abstract class ${record.capitalizedName}SchemaBase extends SparkSchema {
                     .withColumn(${columnVars[$field.name]} + "_MATCHES_SCALE", col(columnPrefix + ${columnVars[$field.name]}).rlike(("^[0-9]*(?:\\.[0-9]{0,${field.getValidation().getScale()}})?$")))
                 #end
                 #if (${field.getValidation().getMinLength()})
-                    .withColumn(${columnVars[$field.name]} + "_GREATER_THAN_MAX_LENGTH", col(columnPrefix + ${columnVars[$field.name]}).rlike(("^.{${field.getValidation().getMinLength()},}")))
+                    .withColumn(${columnVars[$field.name]} + "_GREATER_THAN_OR_EQUAL_TO_MIN_LENGTH", col(columnPrefix + ${columnVars[$field.name]}).rlike(("^.{${field.getValidation().getMinLength()},}")))
                 #end
                 #if (${field.getValidation().getMaxLength()})
-                    .withColumn(${columnVars[$field.name]} + "_LESS_THAN_MAX_LENGTH", col(columnPrefix + ${columnVars[$field.name]}).rlike(("^.{${field.getValidation().getMaxLength()},}")).equalTo(lit(false)))
+                #set($max = ${field.getValidation().getMaxLength()} + 1)
+                    .withColumn(${columnVars[$field.name]} + "_LESS_THAN_OR_EQUAL_TO_MAX_LENGTH", col(columnPrefix + ${columnVars[$field.name]}).rlike(("^.{$max},}")).equalTo(lit(false)))
                 #end
                 #foreach ($format in $field.getValidation().getFormats())
                     #if ($foreach.first)

foundation/foundation-mda/src/main/resources/templates/data-delivery-spark/dictionary.type.base.java.vm

@@ -51,6 +51,8 @@ public abstract class ${dictionaryType.capitalizedName}Base {
         setValue(value);
     }
 
+    public ${dictionaryType.capitalizedName}Base() {}
+
     public void setValue(${dictionaryType.shortType} value) {
     #if ($dictionaryType.hasScaleValidation())
         if (value != null) {

foundation/foundation-mda/src/main/resources/templates/data-delivery-spark/dictionary.type.impl.java.vm

@@ -19,4 +19,8 @@ public class ${dictionaryType.capitalizedName} extends ${dictionaryType.capitali
         super(value);
     }
 
+    public ${dictionaryType.capitalizedName}() {
+        super();
+    }
+
 }

test/test-mda-models/aissemble-test-data-delivery-pyspark-model/src/aissemble_test_data_delivery_pyspark_model/resources/dictionaries/AddressDictionary.json

@@ -12,11 +12,22 @@
     },
     {
       "name": "zipcode",
-      "simpleType": "integer"
+      "simpleType": "string",
+      "validation": {
+        "maxLength": 10,
+        "minLength": 5,
+        "formats": [
+          "^\\d{5}(?:[-\\s]\\d{4})?$"
+        ]
+      }
     },
     {
       "name": "state",
-      "simpleType": "string"
+      "simpleType": "string",
+      "validation": {
+        "maxLength": 2,
+        "minLength": 2
+      }
     }
   ]
 }

test/test-mda-models/aissemble-test-data-delivery-pyspark-model/src/aissemble_test_data_delivery_pyspark_model/resources/records/PersonWithMToOneRelation.json

@@ -16,7 +16,14 @@
       "package":"com.boozallen.aiops.mda.pattern.record",
       "multiplicity":"m-1",
       "name":"Address",
-      "documentation":"Many to One Relation between Person and Address"
+      "documentation":"Many to One Relation between Person and Address",
+      "required": true
+    },
+    {
+      "package":"com.boozallen.aiops.mda.pattern.record",
+      "multiplicity":"m-1",
+      "name":"CustomData",
+      "documentation":"Many to One Relation between Person and CustomData"
     }
   ]
 }

test/test-mda-models/aissemble-test-data-delivery-pyspark-model/src/aissemble_test_data_delivery_pyspark_model/resources/records/PersonWithOneToMRelation.json

@@ -7,7 +7,14 @@
       "package":"com.boozallen.aiops.mda.pattern.record",
       "multiplicity":"1-m",
       "name":"Address",
-      "documentation":"One to Many Relation between Person and Address"
+      "documentation":"One to Many Relation between Person and Address",
+      "required": true
+    },
+    {
+      "package":"com.boozallen.aiops.mda.pattern.record",
+      "multiplicity":"1-m",
+      "name":"CustomData",
+      "documentation":"One to One Relation between Person and CustomData"
     }
   ]
 }

test/test-mda-models/aissemble-test-data-delivery-pyspark-model/src/aissemble_test_data_delivery_pyspark_model/resources/records/PersonWithOneToOneRelation.json

@@ -7,7 +7,17 @@
       "package":"com.boozallen.aiops.mda.pattern.record",
       "multiplicity":"1-1",
       "name":"Address",
-      "documentation":"One to One Relation between Person and Address"
+      "documentation":"One to One Relation between Person and Address",
+      "required": true
+    }
+  ],
+  "fields": [
+    {
+      "name": "test",
+      "type": {
+        "name": "customType",
+        "package": "com.boozallen.aiops.mda.pattern.dictionary"
+      }
     }
   ]
 }

test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/features/record-relation-validation.feature

@@ -0,0 +1,26 @@
+@relation-validation
+Feature: Validate record with relation configuration
+Scenario Outline: Record with valid relation data
+  Given a "Person" record that has a "<multiplicity>" relation to a record "Address"
+  And the "Address" records are valid
+  When validate the "Person" record
+  Then no exception should be thrown
+ Examples:
+  | multiplicity |
+  | 1-1          |
+  | 1-M          |
+  | M-1          |
+
+Scenario Outline: Record with invalid relation data
+  Given a "Person" record that has a "<multiplicity>" relation to a record "Address"
+  And a required "Address" record is "<validity>"
+  When validate the "Person" record
+  Then the validation exception is thrown
+ Examples:
+  | multiplicity | validity          |
+  | 1-1          | missing           |
+  | 1-1          | invalid           |
+  | 1-M          | missing           |
+  | 1-M          | invalid           |
+  | M-1          | missing           |
+  | M-1          | invalid           |

test/test-mda-models/aissemble-test-data-delivery-pyspark-model/tests/features/record_relation.feature

@@ -18,7 +18,7 @@ Scenario Outline: Records with relations can be serialized as JSON strings
 
   Examples:
    | multiplicity |
-   | 1-1          |  
+   | 1-1          |
    | 1-M          |
    | M-1          |