Fix pipeline state update handling (#777)

.gencode_hash.txt

@@ -4,7 +4,7 @@ cde866298a8a54168d3582b50ec93160209d07338d3d6c58d215855bc35cb158  gencode/docs/c
 4809ac52b172f1754cc311928c9b54611ec52837d7cce11ee03789c603cbc655  gencode/docs/config_mapping.html
 08583688b20f892c0b453f41787ac01a46ac601663736bcd6ed6f57be0758e79  gencode/docs/configuration_endpoint.html
 6cf94d6cb600c75cde32a64bd78acb3ed3b54adfad08dbf6bb159b467e8925c9  gencode/docs/configuration_execution.html
-d0f6ea11943f46c955891b3b49a25a9bffcf558fcabb9ed2b0f7667d8413953b  gencode/docs/configuration_pod.html
+6f8a3766b840e96881e3573d594e28864917efb288e068d6c82de55deb576694  gencode/docs/configuration_pod.html
 fe6dabc7b6634449d457df758bcd441e7ecb87d8461c7e1ce26364f6b7f66f7f  gencode/docs/configuration_pubber.html
 69bee6bfcd2f786d95cfd563ecb497296361c8945a1422385f78d25008f38ef1  gencode/docs/event.html
 0a41807b999f80bd283fcee03e6df5845e13b122d9be7c7d4723b19a334b89d5  gencode/docs/event_discovery.html
@@ -13,7 +13,7 @@ d2fe30200f9a1c530d144137a7774b5b97f268aae41185bfcb25de274d399250  gencode/docs/e
 d39b9cbfbdd3f6b5a34396b6a7ab0e7f92a07264ce216d26e57483b22f751f37  gencode/docs/event_system.html
 04a864b8b37b01a3b234ba86e7e123ebaf3c98bba497c2f06d23c4550ae8194e  gencode/docs/event_validation.html
 fa237fe9d96c2809bf9562abdc5c4f7c7d933010beb5c54407caa4453dec30f9  gencode/docs/metadata.html
-959f637f27f736c47411c29e36d1e994933f5c7f9b10aa400a60c0b12630caa3  gencode/docs/monitoring.html
+0aa459d46e8dc4c1f40e574cad107ae212d08b41fdde1660c052a4dd21a197cc  gencode/docs/monitoring.html
 180b32717db748e164a185b163ef9a97aa83d9d6add306283d5b9852d04af947  gencode/docs/persistent_device.html
 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d  gencode/docs/properties.html
 1766f84518a315fe57e4a4bf934c0a386ad61d87091754a6bab097c686c16019  gencode/docs/readme.md
@@ -33,7 +33,7 @@ ce2c747fab0d374987acc51474a52ca5b3d64659d51cffa671d5442b7114339a  gencode/java/u
 c033a4b2c9920a4314801d1fbb7885b375a4bb890344de937ed30baf4f2c08e1  gencode/java/udmi/schema/BlobBlobsetState.java
 b9f903444ab08907e41eb123286434ff3207b1edd01397af3ddefb8475bbdadc  gencode/java/udmi/schema/BlobsetConfig.java
 fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56  gencode/java/udmi/schema/BlobsetState.java
-e28c3899bf2cb08cf456dbca29ccc3d4559c1d2145e252095689b171be82b4c6  gencode/java/udmi/schema/BridgePodConfiguration.java
+3ee2d42e34f36350925595a6904978698c0d92b0b5d417876bd7bf25b0780210  gencode/java/udmi/schema/BridgePodConfiguration.java
 2e7c0912c6f43b8dae9e9149b7361d3640bde4f01de6e13d8e487fe01be2c2c0  gencode/java/udmi/schema/Bucket.java
 0a4f6bcd5065418c1cdc6c05b900b3de31744847d25b6ab6de7aabb1e724710e  gencode/java/udmi/schema/BuildingTranslation.java
 c47b17d70ed7fffae5cb829623088358eca22c8fa5613edd97518bed0b522620  gencode/java/udmi/schema/CapabilityValidationState.java
@@ -54,7 +54,7 @@ e809df42a73ed843977e447fa3333140dcdbe7d09b4e88bf1b8fd52fbaf9ae1a  gencode/java/u
 b01b7a05f21651a01a605466bb80b2e537059dcbb41f33734c851432da57f657  gencode/java/udmi/schema/EndpointConfiguration.java
 dd2eb479a8e93a851c535c8b40fbd62e152bd60e0473f3b23800ec61f798bed0  gencode/java/udmi/schema/Entry.java
 06758aca1e0043ddf343b504030f47bb19260e99a82e2d66f12e86092a2434ca  gencode/java/udmi/schema/Enumerate.java
-05e3443f9a9da29ed561310b13be1d14459d9dfe292a438e42af2fbd2165a606  gencode/java/udmi/schema/Envelope.java
+8a51984458d96d1798d067005902defa57410167a27dcfb0b730ea38a3326011  gencode/java/udmi/schema/Envelope.java
 e9f5c77be81486b6b8c6d88f70f2d50583d8c3fafa2ac09ead80f44b8d5e751e  gencode/java/udmi/schema/Event.java
 9ff3d1819b5a92713b26f56d08c77dcbb467c6ffc10f75744c6b6a227a13319b  gencode/java/udmi/schema/ExecutionConfiguration.java
 2e77a29988e6c17875d3f781be931e3b835bb310639f60d3bb28f24405a86bed  gencode/java/udmi/schema/FamilyDiscoveryConfig.java
@@ -149,7 +149,7 @@ ae3f2f71fc81b1cffc45ffe9d1c179fba8bb09a398a305e7ba42aa8aeabe125a  gencode/python
 14fd646b9a8638b87e4c421c9dadfb7ed2e66ad02b256217423e3b5dd6c39fd1  gencode/python/udmi/schema/configuration_execution.py
 e30f937983f98673b3e67ac1369fe86964d785092964f7e95cd39611f9283d7c  gencode/python/udmi/schema/configuration_pod.py
 ccc43757750379f3c072f019b25b0ea8d970c48dd0b66e98ec2b0bfb1635a952  gencode/python/udmi/schema/configuration_pod_base.py
-11c8841ed5c2a5bcaf4b44c943c8f70fcb5010f1027a025b46300435353b2432  gencode/python/udmi/schema/configuration_pod_bridge.py
+ef61eea743cc2629893b18411636672cdfec0e209e58eb7918b33b43edab5196  gencode/python/udmi/schema/configuration_pod_bridge.py
 bed77c13436a192047a0dcdcaea7c5d7175e99a76c6c40409cce9e232ab5bc12  gencode/python/udmi/schema/configuration_pubber.py
 fbb4b2c04c170c0da5cdd868612429fe920e44b591fcad2522b2e047d580d537  gencode/python/udmi/schema/entry.py
 fcc75eec4263b11b378e30f929687cfb9619c8dfc524863b6bfb11b7e33be241  gencode/python/udmi/schema/enumeration_feature.py

bin/container

@@ -62,8 +62,6 @@ fi
 echo Using GCP project $GCP_PROJECT
 echo Using udmi namespace $UDMI_NAMESPACE
 
-LIBFILE=build/libs/${target}-1.0-SNAPSHOT-all.jar
-
 current_user=$USER@$HOSTNAME
 
 revparse=`git rev-parse HEAD`
@@ -103,6 +101,8 @@ if [[ -n $prep ]]; then
 
     bin/build
 
+    LIBFILE=build/libs/*-1.0-SNAPSHOT-all.jar
+
     build_time=`date --utc -Imin -r $LIBFILE`
     cat <<EOF > var/deployed_version.json
 {
@@ -128,7 +128,8 @@ if [[ -n $push ]]; then
     for file in $TEMPLATES; do
         cp etc/$file tmp/$file
         sed -i tmp/$file \
-            -e "s^@IMAGE-$target@^$ihash^" \
+            -e "s^@IMAGE@^$ihash^" \
+            -e "s^@TARGET@^$target^" \
             -e "s^@UDMI_NAMESPACE@^$UDMI_NAMESPACE^" \
             -e "s^@GCP_PROJECT@^$GCP_PROJECT^"
         if diff etc/$file tmp/$file; then

bin/set_project

@@ -23,9 +23,9 @@ cd $ROOT
 echo Configuring for GCP project $gcp_project...
 
 current_project=$(gcloud config get project)
-quota_project=$(jq -r .quota_project_id $HOME/.config/gcloud/application_default_credentials.json)
+quota_project=$(jq -r .quota_project_id $HOME/.config/gcloud/application_default_credentials.json) || true
 
-if [[ $quota_project != $gcp_project ]]; then
+if [[ $quota_project != $gcp_project && -n $quota_project ]]; then
     echo Setting gcloud quota project to $gcp_project
     gcloud auth application-default set-quota-project $gcp_project
 fi

bin/test_itemcheck

@@ -11,10 +11,10 @@ GOLDEN_SCHEMAS=$UDMI_ROOT/etc/schema_itemized.out
 
 failures=
 
-echo Comparing diff $RESULTS_OUT $GOLDEN_FILE...
+echo Comparing diff $RESULTS_OUT $GOLDEN_FILE
 diff -bu $RESULTS_OUT $GOLDEN_FILE || failures+="results "
 
-echo Comparing diff $SCHEMA_OUT $GOLDEN_SCHEMAS...
+echo Comparing diff $SCHEMA_OUT $GOLDEN_SCHEMAS
 diff -bu $SCHEMA_OUT $GOLDEN_SCHEMAS || failures+="schema "
 
 [[ -z $failures ]] || fail itemized checks failied because of $failures

bin/test_itemized

@@ -117,9 +117,9 @@ while read -u 7 action test_name pubber_opts; do
     mv $SITE_PATH/out $SITE_PATH/out-$test_marker
 
     if [[ -s $SCHEMA_OUT ]]; then
-        cat $SCHEMA_OUT | sed -e "s/^/$test_name /" >> $SCHEMA_ITEMIZED
+        sed -e "s/^/$test_marker $test_name /" < $SCHEMA_OUT >> $SCHEMA_ITEMIZED
     else
-        echo $test_name >> $SCHEMA_ITEMIZED
+        echo $test_marker $test_name >> $SCHEMA_ITEMIZED
     fi
 
 done 7< $INPUT_FILE

common/src/main/java/com/google/udmi/util/GeneralUtils.java

@@ -319,22 +319,30 @@ public static <T> T catchOrElse(Supplier<T> provider, Supplier<T> alternate) {
     return alternate.get();
   }
 
-  public static boolean catchToFalse(Supplier<Boolean> provider) {
+  public static <T> T catchToElse(Supplier<T> provider, Function<Exception, T> alternate) {
     try {
       return provider.get();
     } catch (Exception e) {
-      return false;
+      return alternate.apply(e);
     }
   }
 
-  public static <T> T catchToNull(Supplier<T> provider) {
+  public static <T> T catchToElse(Supplier<T> provider, T alternate) {
     try {
       return provider.get();
     } catch (Exception e) {
-      return null;
+      return alternate;
     }
   }
 
+  public static boolean catchToFalse(Supplier<Boolean> provider) {
+    return catchToElse(provider, false);
+  }
+
+  public static <T> T catchToNull(Supplier<T> provider) {
+    return catchToElse(provider, (T) null);
+  }
+
   public static <U> U mapReplace(U previous, U added) {
     return added;
   }

etc/schema_itemized.out

@@ -1,21 +1,32 @@
-broken_config
-config_logging
-device_config_acked
-feature_enumeration
-gateway_proxy_events
-pointset_remove_point
-pointset_request_extraneous
-pointset_sample_rate
-state_make_model
-system_last_update
-system_last_update RESULT fail schemas event_pointset_stable STABLE 5/5 Schema violations found
-system_last_update RESULT pass schemas event_system_stable STABLE 5/5 All schema validations passed
-system_last_update RESULT pass schemas state_update_stable STABLE 5/5 All schema validations passed
-system_min_loglevel
-system_mode_restart
-too_much_state
-valid_serial_no
-valid_serial_no RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed
-valid_serial_no RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed
-valid_serial_no RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed
-writeback_success
+01 device_config_acked
+02 pointset_request_extraneous
+03 pointset_remove_point
+04 pointset_remove_point
+05 too_much_state
+06 feature_enumeration
+07 valid_serial_no RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed
+07 valid_serial_no RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed
+07 valid_serial_no RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed
+08 writeback_success
+09 writeback_success
+10 pointset_sample_rate
+11 system_mode_restart
+12 config_logging
+13 broken_config
+14 broken_config
+15 broken_config
+16 broken_config
+17 broken_config
+18 system_last_update RESULT fail schemas event_pointset_stable STABLE 5/5 Schema violations found
+18 system_last_update RESULT fail schemas state_update_stable STABLE 5/5 Schema violations found
+18 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 All schema validations passed
+19 state_make_model
+23 valid_serial_no
+24 system_last_update
+25 system_min_loglevel
+26 system_min_loglevel
+30 gateway_proxy_events
+31 gateway_proxy_events
+35 gateway_proxy_events
+36 gateway_proxy_events
+40 device_config_acked

etc/test_itemized.out

@@ -21,13 +21,13 @@
 17 CPBLTY fail system broken_config.logging ALPHA 0/1 Timeout after 30s waiting for log category `system.config.receive` level `DEBUG` to be logged
 17 RESULT fail system broken_config ALPHA 0/5 expected:<system.config.[parse]> but was:<system.config.[apply]>
 18 RESULT pass system system_last_update STABLE 5/5 Sequence complete
-19 RESULT fail system state_make_model BETA 0/5 Pipeline type event error: While processing message REDACTED
+19 RESULT fail system state_make_model BETA 0/5 Timeout waiting for no applicable system status
 23 RESULT skip system valid_serial_no ALPHA 0/0 State testing disabled
 24 RESULT fail system system_last_update STABLE 0/5 Timeout waiting for state last_config matches config timestamp
 25 RESULT fail system system_min_loglevel ALPHA 0/5 Received state update with no-state device
 26 RESULT pass system system_min_loglevel ALPHA 5/5 Sequence complete
 30 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete
 31 RESULT fail gateway gateway_proxy_events ALPHA 0/5 Timeout waiting for Missing data from AHU-22
 35 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete
-36 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete
+36 RESULT fail gateway gateway_proxy_events ALPHA 0/5 Received state update with no-state device
 40 RESULT skip system device_config_acked BETA 0/0 No config check for proxy device

etc/validator.out

@@ -102,9 +102,9 @@ sites/udmi_site_model/out/devices/AHU-1/state.out
   "sub_folder" : "update",
   "sub_type" : "state",
   "status" : {
-    "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold",
-    "detail" : "state_update: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR",
-    "category" : "validation.device.schema",
+    "message" : "Multiple validation errors",
+    "detail" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold; While converting to json node: 1 schema violations found",
+    "category" : "validation.device.multiple",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
@@ -114,6 +114,12 @@ sites/udmi_site_model/out/devices/AHU-1/state.out
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
+  }, {
+    "message" : "While converting to json node: 1 schema violations found",
+    "detail" : "state_update: While converting to json node: 1 schema violations found REDACTED_ERROR; 1 schema violations found; /system: object instance has properties which are not allowed by the schema: [\"extraField\"]",
+    "category" : "validation.device.schema",
+    "timestamp" : "REDACTED_TIMESTAMP",
+    "level" : 500
   } ]
 }
 ::::::::::::::
@@ -316,9 +322,9 @@ sites/udmi_site_model/out/devices/GAT-123/event_invalid.out
   "sub_folder" : "invalid",
   "sub_type" : "event",
   "status" : {
-    "message" : "Multiple validation errors",
-    "detail" : "Unknown schema subFolder 'event_invalid' for GAT-123; While converting to json node: 1 schema violations found",
-    "category" : "validation.device.multiple",
+    "message" : "Unknown schema subFolder 'event_invalid' for GAT-123",
+    "detail" : "event_invalid: Unknown schema subFolder 'event_invalid' for GAT-123 REDACTED_ERROR",
+    "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
   },
@@ -328,12 +334,6 @@ sites/udmi_site_model/out/devices/GAT-123/event_invalid.out
     "category" : "validation.device.schema",
     "timestamp" : "REDACTED_TIMESTAMP",
     "level" : 500
-  }, {
-    "message" : "While converting to json node: 1 schema violations found",
-    "detail" : "event_invalid: While converting to json node: 1 schema violations found REDACTED_ERROR; 1 schema violations found; /subFolder: instance value (\"invalid\") not found in enum (possible values: [\"udmi\",\"update\",\"error\",\"cloud\",\"discovery\",\"mapping\",\"system\",\"gateway\",\"swarm\",\"localnet\",\"pointset\",\"validation\",\"blobset\",\"monitoring\"])",
-    "category" : "validation.device.schema",
-    "timestamp" : "REDACTED_TIMESTAMP",
-    "level" : 500
   } ]
 }
 ::::::::::::::