From 6cf778cb3997d6b287ac2f04036edd27eceda880 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Tue, 14 Nov 2023 11:29:13 -0400 Subject: [PATCH 01/30] Shunt tweaks --- bin/container | 7 ++++--- shunt | 1 + udmis/Dockerfile.shunt | 18 ++++++++++++++++++ udmis/etc/k8s_udmis.yaml | 10 +++++----- 4 files changed, 28 insertions(+), 8 deletions(-) create mode 120000 shunt create mode 100644 udmis/Dockerfile.shunt diff --git a/bin/container b/bin/container index 72f5345f0d..c5df81f550 100755 --- a/bin/container +++ b/bin/container @@ -62,8 +62,6 @@ fi echo Using GCP project $GCP_PROJECT echo Using udmi namespace $UDMI_NAMESPACE -LIBFILE=build/libs/${target}-1.0-SNAPSHOT-all.jar - current_user=$USER@$HOSTNAME revparse=`git rev-parse HEAD` @@ -103,6 +101,8 @@ if [[ -n $prep ]]; then bin/build + LIBFILE=build/libs/*-1.0-SNAPSHOT-all.jar + build_time=`date --utc -Imin -r $LIBFILE` cat < var/deployed_version.json { @@ -128,7 +128,8 @@ if [[ -n $push ]]; then for file in $TEMPLATES; do cp etc/$file tmp/$file sed -i tmp/$file \ - -e "s^@IMAGE-$target@^$ihash^" \ + -e "s^@IMAGE@^$ihash^" \ + -e "s^@TARGET@^$target^" \ -e "s^@UDMI_NAMESPACE@^$UDMI_NAMESPACE^" \ -e "s^@GCP_PROJECT@^$GCP_PROJECT^" if diff etc/$file tmp/$file; then diff --git a/shunt b/shunt new file mode 120000 index 0000000000..4dce139632 --- /dev/null +++ b/shunt @@ -0,0 +1 @@ +udmis \ No newline at end of file diff --git a/udmis/Dockerfile.shunt b/udmis/Dockerfile.shunt new file mode 100644 index 0000000000..f83ddae2ad --- /dev/null +++ b/udmis/Dockerfile.shunt @@ -0,0 +1,18 @@ +FROM alpine:latest + +WORKDIR /root + +RUN apk add openjdk17 bash gcompat mosquitto-clients curl jq + +# Workaround for https://github.com/grpc/grpc-java/issues/8751 +ENV LD_PRELOAD=/lib/libgcompat.so.0 + +ADD build/libs/udmis-1.0-SNAPSHOT-all.jar build/libs/udmis-1.0-SNAPSHOT-all.jar + +ADD bin/ bin/ + +ADD var/ var/ + +ENV CLEARBLADE_CONFIGURATION=/udmi/clearblade.json + +CMD ["/root/bin/run", "var/shunt_pod.json"] diff --git a/udmis/etc/k8s_udmis.yaml b/udmis/etc/k8s_udmis.yaml index 5ce9c4f31f..8a9b511559 100644 --- a/udmis/etc/k8s_udmis.yaml +++ b/udmis/etc/k8s_udmis.yaml @@ -1,24 +1,24 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: udmis-pods + name: @TARGET@-pods spec: selector: matchLabels: - app: udmis + app: @TARGET@ role: master tier: backend replicas: 3 template: metadata: labels: - app: udmis + app: @TARGET@ role: master tier: backend spec: containers: - - name: udmis-core - image: @IMAGE-udmis@ + - name: @TARGET@-core + image: @IMAGE@ imagePullPolicy: Always readinessProbe: exec: From 2203f3967055e42cf6c880e34d71647178acaec7 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 15 Nov 2023 06:00:44 -0400 Subject: [PATCH 02/30] Shunt pod definition --- shunt | 1 - udmis/etc/prod_pod.json | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) delete mode 120000 shunt diff --git a/shunt b/shunt deleted file mode 120000 index 4dce139632..0000000000 --- a/shunt +++ /dev/null @@ -1 +0,0 @@ -udmis \ No newline at end of file diff --git a/udmis/etc/prod_pod.json b/udmis/etc/prod_pod.json index 91618d5c11..e6bff09b12 100644 --- a/udmis/etc/prod_pod.json +++ b/udmis/etc/prod_pod.json @@ -48,5 +48,20 @@ "hostname": "udmis-broker", "port": "1883" } + }, + "bridges": { + "shunt": { + "enabled": "${SHUNT_NAME}", + "from": { + "hostname": "${SHUNT_FROM}", + "recv_id": "${SHUNT_NAME}-take", + "send_id": "${SHUNT_NAME}" + }, + "to": { + "hostname": "${SHUNT_MORF}", + "recv_id": "${SHUNT_NAME}-take", + "send_id": "${SHUNT_NAME}" + } + } } } From cf7c466395d4a91d9ea0c2a0bbc30065065962cc Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 15 Nov 2023 06:20:07 -0400 Subject: [PATCH 03/30] Initial config hashing --- .gencode_hash.txt | 6 +-- gencode/docs/configuration_pod.html | 47 +++++++++++++++++++ .../udmi/schema/BridgePodConfiguration.java | 6 ++- .../udmi/schema/configuration_pod_bridge.py | 4 ++ schema/configuration_pod_bridge.json | 3 ++ udmis/bin/run | 11 +++++ udmis/etc/k8s_config.yaml | 1 + udmis/etc/k8s_udmis.yaml | 6 +++ .../bos/udmi/service/pod/UdmiServicePod.java | 4 ++ .../service/core/BridgeProcessorTest.java | 2 +- 10 files changed, 85 insertions(+), 5 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index 1ae6ea9370..fd95e095ab 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -4,7 +4,7 @@ d3758eba2529d4a5f1dfd5ed3355a536936b02285ddde7cc75b1f41f4916203a gencode/docs/c d40bfc9f4a30c56986435dc08f1e5f42401e5ac043359a1e359011c913cad673 gencode/docs/config_mapping.html 08583688b20f892c0b453f41787ac01a46ac601663736bcd6ed6f57be0758e79 gencode/docs/configuration_endpoint.html 6cf94d6cb600c75cde32a64bd78acb3ed3b54adfad08dbf6bb159b467e8925c9 gencode/docs/configuration_execution.html -d0f6ea11943f46c955891b3b49a25a9bffcf558fcabb9ed2b0f7667d8413953b gencode/docs/configuration_pod.html +122ee86d875c09c36585fc1ee4c7e929c852b9dc2483d36aca0f6fdeb6c01a95 gencode/docs/configuration_pod.html 1fbd42e4591a609d7f8869c6d1942477d96221ee01ee9bf4aae0bc23d570f301 gencode/docs/configuration_pubber.html 96186777da06f95eae1d16d73555445d23608a9301636ea1ccd17922b3fe4019 gencode/docs/event.html 587e048c161273b927de67b899204bf0e183db64e59ae513f833e5eff406b1ab gencode/docs/event_discovery.html @@ -33,7 +33,7 @@ ce2c747fab0d374987acc51474a52ca5b3d64659d51cffa671d5442b7114339a gencode/java/u c033a4b2c9920a4314801d1fbb7885b375a4bb890344de937ed30baf4f2c08e1 gencode/java/udmi/schema/BlobBlobsetState.java b9f903444ab08907e41eb123286434ff3207b1edd01397af3ddefb8475bbdadc gencode/java/udmi/schema/BlobsetConfig.java fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56 gencode/java/udmi/schema/BlobsetState.java -e28c3899bf2cb08cf456dbca29ccc3d4559c1d2145e252095689b171be82b4c6 gencode/java/udmi/schema/BridgePodConfiguration.java +fba3ce9cb65438684dfd9c54777402bc8c76bf114b8a6566829fcf400881ddd2 gencode/java/udmi/schema/BridgePodConfiguration.java 2e7c0912c6f43b8dae9e9149b7361d3640bde4f01de6e13d8e487fe01be2c2c0 gencode/java/udmi/schema/Bucket.java 0a4f6bcd5065418c1cdc6c05b900b3de31744847d25b6ab6de7aabb1e724710e gencode/java/udmi/schema/BuildingTranslation.java 36a165d3c65ca02cd095606a27d7efa21f6d3b7134bb49771b315dcdfa7154b7 gencode/java/udmi/schema/Category.java @@ -148,7 +148,7 @@ ae3f2f71fc81b1cffc45ffe9d1c179fba8bb09a398a305e7ba42aa8aeabe125a gencode/python 14fd646b9a8638b87e4c421c9dadfb7ed2e66ad02b256217423e3b5dd6c39fd1 gencode/python/udmi/schema/configuration_execution.py e30f937983f98673b3e67ac1369fe86964d785092964f7e95cd39611f9283d7c gencode/python/udmi/schema/configuration_pod.py ccc43757750379f3c072f019b25b0ea8d970c48dd0b66e98ec2b0bfb1635a952 gencode/python/udmi/schema/configuration_pod_base.py -11c8841ed5c2a5bcaf4b44c943c8f70fcb5010f1027a025b46300435353b2432 gencode/python/udmi/schema/configuration_pod_bridge.py +23b82dd6008cd848fe40f9019a30fe1395f627b172ddc318af4599d3f3b1381a gencode/python/udmi/schema/configuration_pod_bridge.py bed77c13436a192047a0dcdcaea7c5d7175e99a76c6c40409cce9e232ab5bc12 gencode/python/udmi/schema/configuration_pubber.py fbb4b2c04c170c0da5cdd868612429fe920e44b591fcad2522b2e047d580d537 gencode/python/udmi/schema/entry.py fcc75eec4263b11b378e30f929687cfb9619c8dfc524863b6bfb11b7e33be241 gencode/python/udmi/schema/enumeration_feature.py diff --git a/gencode/docs/configuration_pod.html b/gencode/docs/configuration_pod.html index fd2a77e827..fcd5f07b67 100644 --- a/gencode/docs/configuration_pod.html +++ b/gencode/docs/configuration_pod.html @@ -2008,6 +2008,53 @@

+
+
+
+

+ +

+
+ +
+
+ +
root + + + + bridges + + + + Bridge Pod Configuration + + + + enabled
Type: string
+ + + + + + + +
+
+
+
diff --git a/gencode/java/udmi/schema/BridgePodConfiguration.java b/gencode/java/udmi/schema/BridgePodConfiguration.java index 69fa5f6d95..0ee349a984 100644 --- a/gencode/java/udmi/schema/BridgePodConfiguration.java +++ b/gencode/java/udmi/schema/BridgePodConfiguration.java @@ -16,12 +16,15 @@ */ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({ + "enabled", "from", "to" }) @Generated("jsonschema2pojo") public class BridgePodConfiguration { + @JsonProperty("enabled") + public String enabled; /** * Endpoint Configuration *

@@ -46,6 +49,7 @@ public int hashCode() { int result = 1; result = ((result* 31)+((this.from == null)? 0 :this.from.hashCode())); result = ((result* 31)+((this.to == null)? 0 :this.to.hashCode())); + result = ((result* 31)+((this.enabled == null)? 0 :this.enabled.hashCode())); return result; } @@ -58,7 +62,7 @@ public boolean equals(Object other) { return false; } BridgePodConfiguration rhs = ((BridgePodConfiguration) other); - return (((this.from == rhs.from)||((this.from!= null)&&this.from.equals(rhs.from)))&&((this.to == rhs.to)||((this.to!= null)&&this.to.equals(rhs.to)))); + return ((((this.from == rhs.from)||((this.from!= null)&&this.from.equals(rhs.from)))&&((this.to == rhs.to)||((this.to!= null)&&this.to.equals(rhs.to))))&&((this.enabled == rhs.enabled)||((this.enabled!= null)&&this.enabled.equals(rhs.enabled)))); } } diff --git a/gencode/python/udmi/schema/configuration_pod_bridge.py b/gencode/python/udmi/schema/configuration_pod_bridge.py index 23188e5b55..cb58ab7171 100644 --- a/gencode/python/udmi/schema/configuration_pod_bridge.py +++ b/gencode/python/udmi/schema/configuration_pod_bridge.py @@ -7,6 +7,7 @@ class BridgePodConfiguration: """Generated schema class""" def __init__(self): + self.enabled = None self.from = None self.to = None @@ -15,6 +16,7 @@ def from_dict(source): if not source: return None result = BridgePodConfiguration() + result.enabled = source.get('enabled') result.from = EndpointConfiguration.from_dict(source.get('from')) result.to = EndpointConfiguration.from_dict(source.get('to')) return result @@ -37,6 +39,8 @@ def expand_dict(input): def to_dict(self): result = {} + if self.enabled: + result['enabled'] = self.enabled # 5 if self.from: result['from'] = self.from.to_dict() # 4 if self.to: diff --git a/schema/configuration_pod_bridge.json b/schema/configuration_pod_bridge.json index 097b22311d..eac802d827 100644 --- a/schema/configuration_pod_bridge.json +++ b/schema/configuration_pod_bridge.json @@ -6,6 +6,9 @@ "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": false, "properties": { + "enabled": { + "type": "string" + }, "from": { "$ref": "file:configuration_endpoint.json" }, diff --git a/udmis/bin/run b/udmis/bin/run index 55ad9c7c9c..3be7a2b412 100755 --- a/udmis/bin/run +++ b/udmis/bin/run @@ -44,8 +44,19 @@ export UDMI_PREFIX= if [[ -n $UDMI_NAMESPACE ]]; then UDMI_PREFIX=${UDMI_NAMESPACE}~ echo Using UDMI_PREFIX $UDMI_PREFIX +else + echo No UDMI_NAMESPACE defined, so no UDMI_PREFIX fi +if [[ -n $PIPE_SHUNT ]]; then + readarray -td, a <<<"$PIPE_SHUNT,"; unset 'a[-1]'; declare -p a; + export SHUNT_NAME=$(echo ${a[0]}) + export SHUNT_FROM=$(echo ${a[1]}) + export SHUNT_MORF=$(echo ${a[2]}) + echo Configured udmi shunt $SHUT_NAME between $SHUNT_FROM and $SHUNT_MORF +else + echo No PIPE_SHUNT defined, so no SHUNT_NAME configured. +fi if [[ -n $CLEARBLADE_PROJECT ]]; then echo CLEARBLADE_PROJECT defined, would be clobbered by value from CLEARBLADE_CONFIGURATION diff --git a/udmis/etc/k8s_config.yaml b/udmis/etc/k8s_config.yaml index c30141119e..b43df42e9a 100644 --- a/udmis/etc/k8s_config.yaml +++ b/udmis/etc/k8s_config.yaml @@ -5,3 +5,4 @@ metadata: data: GCP_IOT_OPTIONS: disabled UDMI_NAMESPACE: @UDMI_NAMESPACE@ + PIPE_SHUNT: obd,projectA,projectB diff --git a/udmis/etc/k8s_udmis.yaml b/udmis/etc/k8s_udmis.yaml index 8a9b511559..762c2e351e 100644 --- a/udmis/etc/k8s_udmis.yaml +++ b/udmis/etc/k8s_udmis.yaml @@ -45,6 +45,12 @@ spec: configMapKeyRef: name: config key: GCP_IOT_OPTIONS + - name: UDMI_SHUNT + valueFrom: + configMapKeyRef: + name: config + key: UDMI_SHUNT + optional: true volumes: - name: secret-volume projected: diff --git a/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java b/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java index 1efb70b703..89cad68381 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java @@ -159,6 +159,10 @@ private void createAccess(String name, IotAccess config) { } private void createBridge(String name, BridgePodConfiguration config) { + if (config.enabled != null && config.enabled.isEmpty()) { + warn("Skipping not-enabled bridge " + name); + return; + } EndpointConfiguration from = makeConfig(config.from); EndpointConfiguration to = makeConfig(config.to); putComponent(name, () -> new BridgeProcessor(from, to)); diff --git a/udmis/src/test/java/com/google/bos/udmi/service/core/BridgeProcessorTest.java b/udmis/src/test/java/com/google/bos/udmi/service/core/BridgeProcessorTest.java index 9911563fc9..c83e935b6c 100644 --- a/udmis/src/test/java/com/google/bos/udmi/service/core/BridgeProcessorTest.java +++ b/udmis/src/test/java/com/google/bos/udmi/service/core/BridgeProcessorTest.java @@ -19,7 +19,7 @@ class BridgeProcessorTest extends MessageTestCore { Map> results = new ConcurrentHashMap<>(); @Test - public void basicTest() { + public void basic_bridge() { EndpointConfiguration from = getConfiguration(false, "from"); EndpointConfiguration to = getConfiguration(false, "to"); BridgeProcessor bridgeProcessor = new BridgeProcessor(from, to); From 863750837c81be68d638f4272f892131982287b1 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 15 Nov 2023 08:07:06 -0400 Subject: [PATCH 04/30] updating shunt name --- udmis/etc/k8s_config.yaml | 2 +- udmis/etc/k8s_udmis.yaml | 4 ++-- .../java/com/google/bos/udmi/service/pod/UdmiServicePod.java | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/udmis/etc/k8s_config.yaml b/udmis/etc/k8s_config.yaml index b43df42e9a..6fff588acb 100644 --- a/udmis/etc/k8s_config.yaml +++ b/udmis/etc/k8s_config.yaml @@ -5,4 +5,4 @@ metadata: data: GCP_IOT_OPTIONS: disabled UDMI_NAMESPACE: @UDMI_NAMESPACE@ - PIPE_SHUNT: obd,projectA,projectB + PIPE_SHUNT: obd_shunt,dbo-rews-dev,bos-platform-dev diff --git a/udmis/etc/k8s_udmis.yaml b/udmis/etc/k8s_udmis.yaml index 762c2e351e..c108c930b3 100644 --- a/udmis/etc/k8s_udmis.yaml +++ b/udmis/etc/k8s_udmis.yaml @@ -45,11 +45,11 @@ spec: configMapKeyRef: name: config key: GCP_IOT_OPTIONS - - name: UDMI_SHUNT + - name: PIPE_SHUNT valueFrom: configMapKeyRef: name: config - key: UDMI_SHUNT + key: PIPE_SHUNT optional: true volumes: - name: secret-volume diff --git a/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java b/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java index 89cad68381..21b712b7e0 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java @@ -159,10 +159,12 @@ private void createAccess(String name, IotAccess config) { } private void createBridge(String name, BridgePodConfiguration config) { - if (config.enabled != null && config.enabled.isEmpty()) { + String enabled = variableSubstitution(config.enabled); + if (enabled != null && enabled.isEmpty()) { warn("Skipping not-enabled bridge " + name); return; } + info(format("Creating bridge %s with enabled %s", name, config.enabled)); EndpointConfiguration from = makeConfig(config.from); EndpointConfiguration to = makeConfig(config.to); putComponent(name, () -> new BridgeProcessor(from, to)); From 4804062a72131d7632608027c9ecd055d4c5ca3e Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 15 Nov 2023 08:51:17 -0400 Subject: [PATCH 05/30] Fix diagnostic typo --- udmis/bin/run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/udmis/bin/run b/udmis/bin/run index 3be7a2b412..e1e4a2ad2b 100755 --- a/udmis/bin/run +++ b/udmis/bin/run @@ -53,7 +53,7 @@ if [[ -n $PIPE_SHUNT ]]; then export SHUNT_NAME=$(echo ${a[0]}) export SHUNT_FROM=$(echo ${a[1]}) export SHUNT_MORF=$(echo ${a[2]}) - echo Configured udmi shunt $SHUT_NAME between $SHUNT_FROM and $SHUNT_MORF + echo Configured udmi shunt $SHUNT_NAME between $SHUNT_FROM and $SHUNT_MORF else echo No PIPE_SHUNT defined, so no SHUNT_NAME configured. fi From da29c46db16be5054f0a77637e4a2d72deb96313 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Wed, 15 Nov 2023 09:35:04 -0400 Subject: [PATCH 06/30] Change shunt order --- udmis/etc/k8s_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/udmis/etc/k8s_config.yaml b/udmis/etc/k8s_config.yaml index 6fff588acb..b416404068 100644 --- a/udmis/etc/k8s_config.yaml +++ b/udmis/etc/k8s_config.yaml @@ -5,4 +5,4 @@ metadata: data: GCP_IOT_OPTIONS: disabled UDMI_NAMESPACE: @UDMI_NAMESPACE@ - PIPE_SHUNT: obd_shunt,dbo-rews-dev,bos-platform-dev + PIPE_SHUNT: obd_shunt,bos-platform-dev,dbo-rews-dev From 619eb6f458dd39d470f738b78bf25ae03f2f2a85 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 17 Nov 2023 04:35:25 -0400 Subject: [PATCH 07/30] Testing tweaks --- bin/set_project | 4 ++-- udmis/Dockerfile.udmis | 8 +++++++- udmis/bin/run | 7 ++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/bin/set_project b/bin/set_project index f64ba49066..181c587dd7 100755 --- a/bin/set_project +++ b/bin/set_project @@ -23,9 +23,9 @@ cd $ROOT echo Configuring for GCP project $gcp_project... current_project=$(gcloud config get project) -quota_project=$(jq -r .quota_project_id $HOME/.config/gcloud/application_default_credentials.json) +quota_project=$(jq -r .quota_project_id $HOME/.config/gcloud/application_default_credentials.json) || true -if [[ $quota_project != $gcp_project ]]; then +if [[ $quota_project != $gcp_project && -n $quota_project ]]; then echo Setting gcloud quota project to $gcp_project gcloud auth application-default set-quota-project $gcp_project fi diff --git a/udmis/Dockerfile.udmis b/udmis/Dockerfile.udmis index 8d98bb6de1..7f92620ddf 100644 --- a/udmis/Dockerfile.udmis +++ b/udmis/Dockerfile.udmis @@ -2,7 +2,13 @@ FROM alpine:latest WORKDIR /root -RUN apk add openjdk17 bash gcompat mosquitto-clients curl jq +RUN apk add openjdk17 bash gcompat curl jq python3 git + +RUN curl -sSL https://sdk.cloud.google.com | bash + +ENV PATH=$PATH:/root/google-cloud-sdk/bin + +RUN apk add mosquitto-clients curl jq # Workaround for https://github.com/grpc/grpc-java/issues/8751 ENV LD_PRELOAD=/lib/libgcompat.so.0 diff --git a/udmis/bin/run b/udmis/bin/run index e1e4a2ad2b..923c0619ec 100755 --- a/udmis/bin/run +++ b/udmis/bin/run @@ -73,10 +73,15 @@ if nslookup metadata.google.internal; then echo Configured with GCP service account $GCP_SERVICE_ACCOUNT fi +gcloud config set account $GCP_SERVICE_ACCOUNT + echo echo Final java env: env echo # Run with exec to replace shell so java receives SIGTERM signal. -exec java -jar $POD_JAR $POD_CONFIG +java -jar $POD_JAR $POD_CONFIG || true + +tail -f /dev/null + From 185ef06def3f204ea873a6d81f8707505d36b671 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 17 Nov 2023 04:39:53 -0400 Subject: [PATCH 08/30] Stagger workload --- .github/workflows/testing.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f9d305a1d5..dd485736fe 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -149,6 +149,8 @@ jobs: java-version: '17' - name: base setup run: bin/run_tests install_dependencies + - name: stagger workload + run: sleep $(($MATRIX_SHARD_INDEX * 20 + 20)) - name: registrar clean run: bin/test_regclean $TARGET_PROJECT - name: sequence tests clean From 7a58605577a181ae41dd3a09bb867527666c949a Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 17 Nov 2023 13:49:49 -0400 Subject: [PATCH 09/30] Map service account key --- udmis/etc/k8s_udmis.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/udmis/etc/k8s_udmis.yaml b/udmis/etc/k8s_udmis.yaml index c108c930b3..d00581bc3d 100644 --- a/udmis/etc/k8s_udmis.yaml +++ b/udmis/etc/k8s_udmis.yaml @@ -57,6 +57,8 @@ spec: sources: - secret: name: clearblade.json + - secret: + name: dbo-rews-dev.json - name: tmp emptyDir: medium: Memory From 4dac7976a1ede10dcf1d5e3ceec63d4c54f59057 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 23 Nov 2023 17:23:02 -0400 Subject: [PATCH 10/30] to to morf --- .gencode_hash.txt | 6 +- gencode/docs/configuration_pod.html | 370 +++++++++--------- .../udmi/schema/BridgePodConfiguration.java | 10 +- .../udmi/schema/configuration_pod_bridge.py | 8 +- schema/configuration_pod_bridge.json | 2 +- .../bos/udmi/service/pod/UdmiServicePod.java | 4 +- .../service/core/BridgeProcessorTest.java | 2 +- .../udmi/service/pod/UdmiServicePodTest.java | 12 +- 8 files changed, 207 insertions(+), 207 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index 325fd57514..8067da9422 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -4,7 +4,7 @@ d3758eba2529d4a5f1dfd5ed3355a536936b02285ddde7cc75b1f41f4916203a gencode/docs/c d40bfc9f4a30c56986435dc08f1e5f42401e5ac043359a1e359011c913cad673 gencode/docs/config_mapping.html 08583688b20f892c0b453f41787ac01a46ac601663736bcd6ed6f57be0758e79 gencode/docs/configuration_endpoint.html 6cf94d6cb600c75cde32a64bd78acb3ed3b54adfad08dbf6bb159b467e8925c9 gencode/docs/configuration_execution.html -122ee86d875c09c36585fc1ee4c7e929c852b9dc2483d36aca0f6fdeb6c01a95 gencode/docs/configuration_pod.html +6f8a3766b840e96881e3573d594e28864917efb288e068d6c82de55deb576694 gencode/docs/configuration_pod.html 61c9af0f2000f541a43e0e663fc4b370051aa720f619fe9c1ff5d9bf7cf04bf7 gencode/docs/configuration_pubber.html 96186777da06f95eae1d16d73555445d23608a9301636ea1ccd17922b3fe4019 gencode/docs/event.html 587e048c161273b927de67b899204bf0e183db64e59ae513f833e5eff406b1ab gencode/docs/event_discovery.html @@ -33,7 +33,7 @@ ce2c747fab0d374987acc51474a52ca5b3d64659d51cffa671d5442b7114339a gencode/java/u c033a4b2c9920a4314801d1fbb7885b375a4bb890344de937ed30baf4f2c08e1 gencode/java/udmi/schema/BlobBlobsetState.java b9f903444ab08907e41eb123286434ff3207b1edd01397af3ddefb8475bbdadc gencode/java/udmi/schema/BlobsetConfig.java fcbed49f1af8b791d8c52bcbe18f65521a79d9ac3eb33ec3afd9b342ab2bfc56 gencode/java/udmi/schema/BlobsetState.java -fba3ce9cb65438684dfd9c54777402bc8c76bf114b8a6566829fcf400881ddd2 gencode/java/udmi/schema/BridgePodConfiguration.java +3ee2d42e34f36350925595a6904978698c0d92b0b5d417876bd7bf25b0780210 gencode/java/udmi/schema/BridgePodConfiguration.java 2e7c0912c6f43b8dae9e9149b7361d3640bde4f01de6e13d8e487fe01be2c2c0 gencode/java/udmi/schema/Bucket.java 0a4f6bcd5065418c1cdc6c05b900b3de31744847d25b6ab6de7aabb1e724710e gencode/java/udmi/schema/BuildingTranslation.java 36a165d3c65ca02cd095606a27d7efa21f6d3b7134bb49771b315dcdfa7154b7 gencode/java/udmi/schema/Category.java @@ -148,7 +148,7 @@ ae3f2f71fc81b1cffc45ffe9d1c179fba8bb09a398a305e7ba42aa8aeabe125a gencode/python 14fd646b9a8638b87e4c421c9dadfb7ed2e66ad02b256217423e3b5dd6c39fd1 gencode/python/udmi/schema/configuration_execution.py e30f937983f98673b3e67ac1369fe86964d785092964f7e95cd39611f9283d7c gencode/python/udmi/schema/configuration_pod.py ccc43757750379f3c072f019b25b0ea8d970c48dd0b66e98ec2b0bfb1635a952 gencode/python/udmi/schema/configuration_pod_base.py -23b82dd6008cd848fe40f9019a30fe1395f627b172ddc318af4599d3f3b1381a gencode/python/udmi/schema/configuration_pod_bridge.py +ef61eea743cc2629893b18411636672cdfec0e209e58eb7918b33b43edab5196 gencode/python/udmi/schema/configuration_pod_bridge.py bed77c13436a192047a0dcdcaea7c5d7175e99a76c6c40409cce9e232ab5bc12 gencode/python/udmi/schema/configuration_pubber.py fbb4b2c04c170c0da5cdd868612429fe920e44b591fcad2522b2e047d580d537 gencode/python/udmi/schema/entry.py fcc75eec4263b11b378e30f929687cfb9619c8dfc524863b6bfb11b7e33be241 gencode/python/udmi/schema/enumeration_feature.py diff --git a/gencode/docs/configuration_pod.html b/gencode/docs/configuration_pod.html index fcd5f07b67..22c2bd2b53 100644 --- a/gencode/docs/configuration_pod.html +++ b/gencode/docs/configuration_pod.html @@ -3150,18 +3150,18 @@

-
+
-
+

- +

-
+
root @@ -3185,7 +3185,7 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to

Type: object
+ morf
Type: object

Parameters to define a message endpoint

 @@ -3196,18 +3196,18 @@

-
+
-
+

- +

-
+
root @@ -3231,15 +3231,15 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - protocol

Type: enum (of string)
-Type: enum (of string)
+

Must be one of:

  • "local"
  • "pubsub"
  • "file"
  • "trace"
  • "mqtt"
@@ -3253,18 +3253,18 @@

Must be one of:

-
+
-
+

- +

-
+
root @@ -3288,15 +3288,15 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - transport

Type: enum (of string)
-Type: enum (of string)
+

Must be one of:

  • "ssl"
  • "tcp"
@@ -3310,18 +3310,18 @@

Must be one of:

-
+
-
+

- +

-
+
root @@ -3345,14 +3345,14 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - hostname

Type: string
+ hostname
Type: string
@@ -3364,18 +3364,18 @@

-
+
-
+

- +

-
+
root @@ -3399,14 +3399,14 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - error

Type: string
+ error
Type: string

Error message container for capturing errors during parsing/handling

 @@ -3419,18 +3419,18 @@

-
+
-
+

- +

-
+
root @@ -3454,14 +3454,14 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - port

Type: integer Default: 8883
+ port
Type: integer Default: 8883
@@ -3473,18 +3473,18 @@

-
+
-
+

- +

-
+
root @@ -3508,14 +3508,14 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - config_sync_sec

Type: integer
+ config_sync_sec
Type: integer

Delay waiting for config message on start, 0 for default, <0 to disable

 @@ -3528,18 +3528,18 @@

-
+
-
+

- +

-
+
root @@ -3563,14 +3563,14 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - client_id

Type: string
+ client_id
Type: string
@@ -3582,18 +3582,18 @@

-
+
-
+

- +

-
+
root @@ -3617,16 +3617,16 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - msg_prefix

Type: string
+ msg_prefix
Type: string

Prefix for message topics/addresses

-Must match regular expression: ^[-_/a-zA-Z0-9]+$ +Must match regular expression: ^[-_/a-zA-Z0-9]+$ @@ -3637,18 +3637,18 @@

-
+
-
+

- +

-
+
root @@ -3672,16 +3672,16 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - recv_id

Type: string
+ recv_id
Type: string

Id for the receiving message channel

-Must match regular expression: ^[-_/a-zA-Z0-9#]+$ +Must match regular expression: ^[-_/a-zA-Z0-9#]+$ @@ -3692,18 +3692,18 @@

-
+
-
+

- +

-
+
root @@ -3727,16 +3727,16 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - send_id

Type: string
+ send_id
Type: string

Id for the sending messages channel

-Must match regular expression: ^[-_/a-zA-Z0-9#]+$ +Must match regular expression: ^[-_/a-zA-Z0-9#]+$ @@ -3747,18 +3747,18 @@

-
+
-
+

- +

-
+
root @@ -3782,16 +3782,16 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - distributor

Type: string
+ distributor
Type: string

processor designation for a distributor channel

-Must match regular expression: ^[a-z][a-z0-9]*(_[a-z0-9]+)*$ +Must match regular expression: ^[a-z][a-z0-9]*(_[a-z0-9]+)*$ @@ -3802,18 +3802,18 @@

-
+
-
+

- +

-
+
root @@ -3837,14 +3837,14 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - auth_provider

Type: object
+ auth_provider
Type: object
No Additional Properties @@ -3852,18 +3852,18 @@

-
+
-
+

- +

-
+
root @@ -3887,21 +3887,21 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - auth_provider + auth_provider - basic

Type: object
+ basic
Type: object
No Additional Properties @@ -3909,18 +3909,18 @@

-
+
-
+

- +

-
+
root @@ -3944,28 +3944,28 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - auth_provider + auth_provider - basic + basic - username

Type: string
+ username
Type: string
@@ -3977,18 +3977,18 @@

-
+
-
+

- +

-
+
root @@ -4012,28 +4012,28 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - auth_provider + auth_provider - basic + basic - password

Type: string
+ password
Type: string
@@ -4049,18 +4049,18 @@

-
+
-
+

- +

-
+
root @@ -4084,21 +4084,21 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - auth_provider + auth_provider - jwt

Type: object
+ jwt
Type: object
No Additional Properties @@ -4106,18 +4106,18 @@

-
+
-
+

- +

-
+
root @@ -4141,28 +4141,28 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - auth_provider + auth_provider - jwt + jwt - audience

Type: string
+ audience
Type: string
@@ -4182,18 +4182,18 @@

-
+
-
+

- +

-
+
root @@ -4217,14 +4217,14 @@

d="M4 8a.5.5 0 0 1 .5-.5h5.793L8.146 5.354a.5.5 0 1 1 .708-.708l3 3a.5.5 0 0 1 0 .708l-3 3a.5.5 0 0 1-.708-.708L10.293 8.5H4.5A.5.5 0 0 1 4 8z" /> - to + morf - generation

Type: string
+ generation
Type: string

The timestamp of the endpoint generation

 @@ -4234,7 +4234,7 @@


Example:
-
"2019-01-17T14:02:29.364Z"
+
"2019-01-17T14:02:29.364Z"
diff --git a/gencode/java/udmi/schema/BridgePodConfiguration.java b/gencode/java/udmi/schema/BridgePodConfiguration.java index 0ee349a984..977baf3b1c 100644 --- a/gencode/java/udmi/schema/BridgePodConfiguration.java +++ b/gencode/java/udmi/schema/BridgePodConfiguration.java @@ -18,7 +18,7 @@ @JsonPropertyOrder({ "enabled", "from", - "to" + "morf" }) @Generated("jsonschema2pojo") public class BridgePodConfiguration { @@ -40,15 +40,15 @@ public class BridgePodConfiguration { * Parameters to define a message endpoint * */ - @JsonProperty("to") + @JsonProperty("morf") @JsonPropertyDescription("Parameters to define a message endpoint") - public EndpointConfiguration to; + public EndpointConfiguration morf; @Override public int hashCode() { int result = 1; result = ((result* 31)+((this.from == null)? 0 :this.from.hashCode())); - result = ((result* 31)+((this.to == null)? 0 :this.to.hashCode())); + result = ((result* 31)+((this.morf == null)? 0 :this.morf.hashCode())); result = ((result* 31)+((this.enabled == null)? 0 :this.enabled.hashCode())); return result; } @@ -62,7 +62,7 @@ public boolean equals(Object other) { return false; } BridgePodConfiguration rhs = ((BridgePodConfiguration) other); - return ((((this.from == rhs.from)||((this.from!= null)&&this.from.equals(rhs.from)))&&((this.to == rhs.to)||((this.to!= null)&&this.to.equals(rhs.to))))&&((this.enabled == rhs.enabled)||((this.enabled!= null)&&this.enabled.equals(rhs.enabled)))); + return ((((this.from == rhs.from)||((this.from!= null)&&this.from.equals(rhs.from)))&&((this.morf == rhs.morf)||((this.morf!= null)&&this.morf.equals(rhs.morf))))&&((this.enabled == rhs.enabled)||((this.enabled!= null)&&this.enabled.equals(rhs.enabled)))); } } diff --git a/gencode/python/udmi/schema/configuration_pod_bridge.py b/gencode/python/udmi/schema/configuration_pod_bridge.py index cb58ab7171..db28c94d58 100644 --- a/gencode/python/udmi/schema/configuration_pod_bridge.py +++ b/gencode/python/udmi/schema/configuration_pod_bridge.py @@ -9,7 +9,7 @@ class BridgePodConfiguration: def __init__(self): self.enabled = None self.from = None - self.to = None + self.morf = None @staticmethod def from_dict(source): @@ -18,7 +18,7 @@ def from_dict(source): result = BridgePodConfiguration() result.enabled = source.get('enabled') result.from = EndpointConfiguration.from_dict(source.get('from')) - result.to = EndpointConfiguration.from_dict(source.get('to')) + result.morf = EndpointConfiguration.from_dict(source.get('morf')) return result @staticmethod @@ -43,6 +43,6 @@ def to_dict(self): result['enabled'] = self.enabled # 5 if self.from: result['from'] = self.from.to_dict() # 4 - if self.to: - result['to'] = self.to.to_dict() # 4 + if self.morf: + result['morf'] = self.morf.to_dict() # 4 return result diff --git a/schema/configuration_pod_bridge.json b/schema/configuration_pod_bridge.json index eac802d827..059f4cd71d 100644 --- a/schema/configuration_pod_bridge.json +++ b/schema/configuration_pod_bridge.json @@ -12,7 +12,7 @@ "from": { "$ref": "file:configuration_endpoint.json" }, - "to": { + "morf": { "$ref": "file:configuration_endpoint.json" } } diff --git a/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java b/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java index 21b712b7e0..41f01bd798 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/pod/UdmiServicePod.java @@ -166,8 +166,8 @@ private void createBridge(String name, BridgePodConfiguration config) { } info(format("Creating bridge %s with enabled %s", name, config.enabled)); EndpointConfiguration from = makeConfig(config.from); - EndpointConfiguration to = makeConfig(config.to); - putComponent(name, () -> new BridgeProcessor(from, to)); + EndpointConfiguration morf = makeConfig(config.morf); + putComponent(name, () -> new BridgeProcessor(from, morf)); } private void createDistributor(String name, EndpointConfiguration config) { diff --git a/udmis/src/test/java/com/google/bos/udmi/service/core/BridgeProcessorTest.java b/udmis/src/test/java/com/google/bos/udmi/service/core/BridgeProcessorTest.java index c83e935b6c..8d4106e7bc 100644 --- a/udmis/src/test/java/com/google/bos/udmi/service/core/BridgeProcessorTest.java +++ b/udmis/src/test/java/com/google/bos/udmi/service/core/BridgeProcessorTest.java @@ -19,7 +19,7 @@ class BridgeProcessorTest extends MessageTestCore { Map> results = new ConcurrentHashMap<>(); @Test - public void basic_bridge() { + public void basicBridge() { EndpointConfiguration from = getConfiguration(false, "from"); EndpointConfiguration to = getConfiguration(false, "to"); BridgeProcessor bridgeProcessor = new BridgeProcessor(from, to); diff --git a/udmis/src/test/java/com/google/bos/udmi/service/pod/UdmiServicePodTest.java b/udmis/src/test/java/com/google/bos/udmi/service/pod/UdmiServicePodTest.java index de2d5ec42f..fd0aa22424 100644 --- a/udmis/src/test/java/com/google/bos/udmi/service/pod/UdmiServicePodTest.java +++ b/udmis/src/test/java/com/google/bos/udmi/service/pod/UdmiServicePodTest.java @@ -116,21 +116,21 @@ public void bridgeTest() throws Exception { EndpointConfiguration reversedFrom = combineConfig(podConfig.flow_defaults, reverseFlow(podConfig.bridges.get("test").from)); final MessageDispatcherImpl fromDispatcher = MessagePipeTestBase.getDispatcherFor(reversedFrom); - EndpointConfiguration reversedTo = - combineConfig(podConfig.flow_defaults, reverseFlow(podConfig.bridges.get("test").to)); - final MessageDispatcherImpl toDispatcher = MessagePipeTestBase.getDispatcherFor(reversedTo); + EndpointConfiguration reversedMorf = + combineConfig(podConfig.flow_defaults, reverseFlow(podConfig.bridges.get("test").morf)); + final MessageDispatcherImpl morfDispatcher = MessagePipeTestBase.getDispatcherFor(reversedMorf); CompletableFuture received = new CompletableFuture<>(); fromDispatcher.registerHandler(LocalnetModel.class, received::complete); BlockingQueue defaulted = new LinkedBlockingQueue<>(); - toDispatcher.registerHandler(Object.class, defaulted::add); + morfDispatcher.registerHandler(Object.class, defaulted::add); pod.activate(); fromDispatcher.activate(); - toDispatcher.activate(); + morfDispatcher.activate(); fromDispatcher.publish(new StateUpdate()); - toDispatcher.publish(new LocalnetModel()); + morfDispatcher.publish(new LocalnetModel()); Object polled = defaulted.poll(RECEIVE_TIMEOUT_SEC, TimeUnit.SECONDS); assertTrue(polled instanceof StateUpdate, "expected pointset state in default"); From 854e9ae244de4371ef1b47cc9a2a707c31ca2ae2 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 23 Nov 2023 19:40:01 -0400 Subject: [PATCH 11/30] Test setup --- bin/test_sequcheck | 13 +++++++------ udmis/Dockerfile.shunt | 18 ------------------ udmis/bin/run | 3 +-- udmis/etc/k8s_config.yaml | 2 +- udmis/etc/prod_pod.json | 6 +++--- 5 files changed, 12 insertions(+), 30 deletions(-) delete mode 100644 udmis/Dockerfile.shunt diff --git a/bin/test_sequcheck b/bin/test_sequcheck index 29f5da6a99..6b1af6de60 100755 --- a/bin/test_sequcheck +++ b/bin/test_sequcheck @@ -3,28 +3,29 @@ suffix=$1 shift || true +RUN_ROOT=$PWD UDMI_ROOT=$(dirname $0)/.. -cd $UDMI_ROOT +mkdir -p out source $UDMI_ROOT/etc/shell_common.sh if [[ -z $suffix ]]; then - bin/sequencer_cache -t + $UDMI_ROOT/bin/sequencer_cache -t fi echo Comparing diff out/sequencer.out etc/sequencer${suffix}.out -diff -u out/sequencer.out etc/sequencer${suffix}.out +diff -u out/sequencer.out $UDMI_ROOT/etc/sequencer${suffix}.out echo Comparing diff out/schema.out etc/schema${suffix}.out -diff -u out/schema.out etc/schema${suffix}.out +diff -u out/schema.out $UDMI_ROOT/etc/schema${suffix}.out [[ -n $suffix ]] && echo Done with suffix test checks. && exit 0 -cp docs/specs/sequences/generated.md out/generated.tmp # Save for test/comparison +cp $UDMI_ROOT/docs/specs/sequences/generated.md out/generated.tmp # Save for test/comparison bin/gencode_seq echo Comparing diff out/generated.tmp docs/specs/sequences/generated.md -diff -u out/generated.tmp docs/specs/sequences/generated.md || ( +diff -u out/generated.tmp $UDMI_ROOT/docs/specs/sequences/generated.md || ( echo Run bin/gencode_seq to regenerate output, if necessary. false ) diff --git a/udmis/Dockerfile.shunt b/udmis/Dockerfile.shunt deleted file mode 100644 index f83ddae2ad..0000000000 --- a/udmis/Dockerfile.shunt +++ /dev/null @@ -1,18 +0,0 @@ -FROM alpine:latest - -WORKDIR /root - -RUN apk add openjdk17 bash gcompat mosquitto-clients curl jq - -# Workaround for https://github.com/grpc/grpc-java/issues/8751 -ENV LD_PRELOAD=/lib/libgcompat.so.0 - -ADD build/libs/udmis-1.0-SNAPSHOT-all.jar build/libs/udmis-1.0-SNAPSHOT-all.jar - -ADD bin/ bin/ - -ADD var/ var/ - -ENV CLEARBLADE_CONFIGURATION=/udmi/clearblade.json - -CMD ["/root/bin/run", "var/shunt_pod.json"] diff --git a/udmis/bin/run b/udmis/bin/run index 923c0619ec..09c240fdad 100755 --- a/udmis/bin/run +++ b/udmis/bin/run @@ -50,7 +50,7 @@ fi if [[ -n $PIPE_SHUNT ]]; then readarray -td, a <<<"$PIPE_SHUNT,"; unset 'a[-1]'; declare -p a; - export SHUNT_NAME=$(echo ${a[0]}) + export SHUNT_NAME=${UDMI_PREFIX}$(echo ${a[0]}) export SHUNT_FROM=$(echo ${a[1]}) export SHUNT_MORF=$(echo ${a[2]}) echo Configured udmi shunt $SHUNT_NAME between $SHUNT_FROM and $SHUNT_MORF @@ -84,4 +84,3 @@ echo java -jar $POD_JAR $POD_CONFIG || true tail -f /dev/null - diff --git a/udmis/etc/k8s_config.yaml b/udmis/etc/k8s_config.yaml index b416404068..d4008d1094 100644 --- a/udmis/etc/k8s_config.yaml +++ b/udmis/etc/k8s_config.yaml @@ -5,4 +5,4 @@ metadata: data: GCP_IOT_OPTIONS: disabled UDMI_NAMESPACE: @UDMI_NAMESPACE@ - PIPE_SHUNT: obd_shunt,bos-platform-dev,dbo-rews-dev + PIPE_SHUNT: mapping_engine,essential-keep-197822,udmi-external diff --git a/udmis/etc/prod_pod.json b/udmis/etc/prod_pod.json index e6bff09b12..e771e298d0 100644 --- a/udmis/etc/prod_pod.json +++ b/udmis/etc/prod_pod.json @@ -55,12 +55,12 @@ "from": { "hostname": "${SHUNT_FROM}", "recv_id": "${SHUNT_NAME}-take", - "send_id": "${SHUNT_NAME}" + "send_id": "${SHUNT_NAME}-put" }, - "to": { + "morf": { "hostname": "${SHUNT_MORF}", "recv_id": "${SHUNT_NAME}-take", - "send_id": "${SHUNT_NAME}" + "send_id": "${SHUNT_NAME}-put" } } } From f313be6c306ff5ebb9e30d76f3e6b4c88134c57e Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Thu, 23 Nov 2023 19:54:53 -0400 Subject: [PATCH 12/30] Revert sequcheck --- bin/test_sequcheck | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/bin/test_sequcheck b/bin/test_sequcheck index 6b1af6de60..29f5da6a99 100755 --- a/bin/test_sequcheck +++ b/bin/test_sequcheck @@ -3,29 +3,28 @@ suffix=$1 shift || true -RUN_ROOT=$PWD UDMI_ROOT=$(dirname $0)/.. -mkdir -p out +cd $UDMI_ROOT source $UDMI_ROOT/etc/shell_common.sh if [[ -z $suffix ]]; then - $UDMI_ROOT/bin/sequencer_cache -t + bin/sequencer_cache -t fi echo Comparing diff out/sequencer.out etc/sequencer${suffix}.out -diff -u out/sequencer.out $UDMI_ROOT/etc/sequencer${suffix}.out +diff -u out/sequencer.out etc/sequencer${suffix}.out echo Comparing diff out/schema.out etc/schema${suffix}.out -diff -u out/schema.out $UDMI_ROOT/etc/schema${suffix}.out +diff -u out/schema.out etc/schema${suffix}.out [[ -n $suffix ]] && echo Done with suffix test checks. && exit 0 -cp $UDMI_ROOT/docs/specs/sequences/generated.md out/generated.tmp # Save for test/comparison +cp docs/specs/sequences/generated.md out/generated.tmp # Save for test/comparison bin/gencode_seq echo Comparing diff out/generated.tmp docs/specs/sequences/generated.md -diff -u out/generated.tmp $UDMI_ROOT/docs/specs/sequences/generated.md || ( +diff -u out/generated.tmp docs/specs/sequences/generated.md || ( echo Run bin/gencode_seq to regenerate output, if necessary. false ) From ea484f3ef9fcec6b6cb899607bf6dedf66aea856 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sun, 26 Nov 2023 06:32:23 -0400 Subject: [PATCH 13/30] FIx unit tests --- .gencode_hash.txt | 4 +- .../com/google/udmi/util/GeneralUtils.java | 16 +++-- gencode/docs/monitoring.html | 4 +- gencode/java/udmi/schema/Envelope.java | 2 + schema/envelope.json | 2 + .../messaging/impl/LocalMessagePipe.java | 12 ++-- .../service/messaging/impl/MessageBase.java | 66 ++++++++++++------- .../messaging/impl/TraceMessagePipe.java | 34 ++-------- udmis/src/test/configs/bridge_pod.json | 2 +- udmis/src/test/configs/trace_pod.json | 2 +- .../messaging/impl/TraceMessagePipeTest.java | 3 +- 11 files changed, 78 insertions(+), 69 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index f47265686d..4232707705 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -13,7 +13,7 @@ d2fe30200f9a1c530d144137a7774b5b97f268aae41185bfcb25de274d399250 gencode/docs/e 6849a96fe7608dc5ebda0dc9595dcea57757bc7c5253c30c30409469b6260498 gencode/docs/event_system.html 816481f69d3b1bdeb2224eaad6e3751a991d20eb98294d89f888b1323505209c gencode/docs/event_validation.html fa237fe9d96c2809bf9562abdc5c4f7c7d933010beb5c54407caa4453dec30f9 gencode/docs/metadata.html -c4fa2845c5ad385a619ec97827370988c059567cb23c8da6894331fed89fefce gencode/docs/monitoring.html +558f16686ce46ee8ec232e78b88e6f694dff23e8f02b5fe967e413d9badb3910 gencode/docs/monitoring.html 180b32717db748e164a185b163ef9a97aa83d9d6add306283d5b9852d04af947 gencode/docs/persistent_device.html 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d gencode/docs/properties.html 1766f84518a315fe57e4a4bf934c0a386ad61d87091754a6bab097c686c16019 gencode/docs/readme.md @@ -53,7 +53,7 @@ e809df42a73ed843977e447fa3333140dcdbe7d09b4e88bf1b8fd52fbaf9ae1a gencode/java/u b01b7a05f21651a01a605466bb80b2e537059dcbb41f33734c851432da57f657 gencode/java/udmi/schema/EndpointConfiguration.java dd2eb479a8e93a851c535c8b40fbd62e152bd60e0473f3b23800ec61f798bed0 gencode/java/udmi/schema/Entry.java 06758aca1e0043ddf343b504030f47bb19260e99a82e2d66f12e86092a2434ca gencode/java/udmi/schema/Enumerate.java -05e3443f9a9da29ed561310b13be1d14459d9dfe292a438e42af2fbd2165a606 gencode/java/udmi/schema/Envelope.java +351ec1178006a546be1218eea6ea604f4b4160ec2db3a71c7095443d546ff9f5 gencode/java/udmi/schema/Envelope.java e9f5c77be81486b6b8c6d88f70f2d50583d8c3fafa2ac09ead80f44b8d5e751e gencode/java/udmi/schema/Event.java 9ff3d1819b5a92713b26f56d08c77dcbb467c6ffc10f75744c6b6a227a13319b gencode/java/udmi/schema/ExecutionConfiguration.java 2e77a29988e6c17875d3f781be931e3b835bb310639f60d3bb28f24405a86bed gencode/java/udmi/schema/FamilyDiscoveryConfig.java diff --git a/common/src/main/java/com/google/udmi/util/GeneralUtils.java b/common/src/main/java/com/google/udmi/util/GeneralUtils.java index a1a655e3ad..108dcc80d9 100644 --- a/common/src/main/java/com/google/udmi/util/GeneralUtils.java +++ b/common/src/main/java/com/google/udmi/util/GeneralUtils.java @@ -315,22 +315,30 @@ public static T catchOrElse(Supplier provider, Supplier alternate) { return alternate.get(); } - public static boolean catchToFalse(Supplier provider) { + public static T catchToElse(Supplier provider, Function alternate) { try { return provider.get(); } catch (Exception e) { - return false; + return alternate.apply(e); } } - public static T catchToNull(Supplier provider) { + public static T catchToElse(Supplier provider, T alternate) { try { return provider.get(); } catch (Exception e) { - return null; + return alternate; } } + public static boolean catchToFalse(Supplier provider) { + return catchToElse(provider, false); + } + + public static T catchToNull(Supplier provider) { + return catchToElse(provider, (T) null); + } + public static U mapReplace(U previous, U added) { return added; } diff --git a/gencode/docs/monitoring.html b/gencode/docs/monitoring.html index 57abdfbfad..708d55563b 100644 --- a/gencode/docs/monitoring.html +++ b/gencode/docs/monitoring.html @@ -3879,7 +3879,7 @@

subFolderType: enum (of string)

Must be one of:

-
  • "udmi"
  • "update"
  • "error"
  • "cloud"
  • "discovery"
  • "mapping"
  • "system"
  • "gateway"
  • "swarm"
  • "localnet"
  • "pointset"
  • "validation"
  • "blobset"
  • "monitoring"
+
  • "udmi"
  • "update"
  • "unknown"
  • "error"
  • "cloud"
  • "discovery"
  • "mapping"
  • "system"
  • "gateway"
  • "swarm"
  • "localnet"
  • "pointset"
  • "validation"
  • "blobset"
  • "monitoring"
@@ -3929,7 +3929,7 @@

subTypeType: enum (of string)

Must be one of:

-
  • "event"
  • "command"
  • "config"
  • "state"
  • "query"
  • "reply"
  • "model"
+
  • "unknown"
  • "event"
  • "command"
  • "config"
  • "state"
  • "query"
  • "reply"
  • "model"
diff --git a/gencode/java/udmi/schema/Envelope.java b/gencode/java/udmi/schema/Envelope.java index 884674e178..ef6b7b7df6 100644 --- a/gencode/java/udmi/schema/Envelope.java +++ b/gencode/java/udmi/schema/Envelope.java @@ -116,6 +116,7 @@ public enum SubFolder { UDMI("udmi"), UPDATE("update"), + UNKNOWN("unknown"), ERROR("error"), CLOUD("cloud"), DISCOVERY("discovery"), @@ -166,6 +167,7 @@ public static Envelope.SubFolder fromValue(String value) { @Generated("jsonschema2pojo") public enum SubType { + UNKNOWN("unknown"), EVENT("event"), COMMAND("command"), CONFIG("config"), diff --git a/schema/envelope.json b/schema/envelope.json index 6a1842eba0..9a6427993c 100644 --- a/schema/envelope.json +++ b/schema/envelope.json @@ -46,6 +46,7 @@ "enum": [ "udmi", "update", + "unknown", "error", "cloud", "discovery", @@ -62,6 +63,7 @@ }, "subType": { "enum": [ + "unknown", "event", "command", "config", diff --git a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/LocalMessagePipe.java b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/LocalMessagePipe.java index 1256959691..de3d9766f5 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/LocalMessagePipe.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/LocalMessagePipe.java @@ -6,7 +6,6 @@ import com.google.bos.udmi.service.messaging.MessagePipe; import java.util.Map; -import java.util.Objects; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; @@ -46,6 +45,10 @@ public static MessagePipe fromConfig(EndpointConfiguration config) { return new LocalMessagePipe(config); } + public static void resetForTestStatic() { + NAMESPACES.clear(); + } + private BlockingQueue getQueueForScope(String name) { checkNotNull(name, "pipe name is null"); Map> namedQueues = @@ -75,13 +78,10 @@ public void resetForTest() { resetForTestStatic(); } - public static void resetForTestStatic() { - NAMESPACES.clear(); - } - @Override public String toString() { String isActive = isActive() ? "*" : "O"; - return format("%s >-%s-> %s", super.toString(), isActive, queueIdentifier(destinationQueue)); + return format("%s >-%s-> %s", super.toString(), isActive, + queueIdentifierStatic(destinationQueue)); } } diff --git a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java index 2b61e193e1..b93474d896 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java @@ -1,14 +1,15 @@ package com.google.bos.udmi.service.messaging.impl; -import static com.google.bos.udmi.service.pod.UdmiServicePod.HOSTNAME; import static com.google.udmi.util.Common.SUBFOLDER_PROPERTY_KEY; +import static com.google.udmi.util.Common.SUBTYPE_PROPERTY_KEY; +import static com.google.udmi.util.GeneralUtils.catchToElse; import static com.google.udmi.util.GeneralUtils.deepCopy; import static com.google.udmi.util.GeneralUtils.friendlyStackTrace; import static com.google.udmi.util.GeneralUtils.ifNotNullGet; import static com.google.udmi.util.GeneralUtils.ifNotNullThen; import static com.google.udmi.util.GeneralUtils.mergeObject; import static com.google.udmi.util.GeneralUtils.stackTraceString; -import static com.google.udmi.util.JsonUtil.convertToStrict; +import static com.google.udmi.util.JsonUtil.convertTo; import static com.google.udmi.util.JsonUtil.fromString; import static com.google.udmi.util.JsonUtil.parseJson; import static com.google.udmi.util.JsonUtil.stringify; @@ -48,14 +49,13 @@ public abstract class MessageBase extends ContainerBase implements MessagePipe { public static final String INVALID_ENVELOPE_KEY = "invalid"; + public static final int EXECUTION_THREADS = 4; + public static final String ERROR_MESSAGE_MARKER = "error-mark"; static final String TERMINATE_MARKER = "terminate"; private static final String DEFAULT_NAMESPACE = "default-namespace"; private static final Set HANDLED_QUEUES = new HashSet<>(); private static final long DEFAULT_POLL_TIME_SEC = 1; private static final long AWAIT_TERMINATION_SEC = 10; - public static final int EXECUTION_THREADS = 4; - public static final String ERROR_MESSAGE_MARKER = "error-mark"; - private final ExecutorService executor = Executors.newFixedThreadPool(EXECUTION_THREADS); private BlockingQueue sourceQueue; private Consumer dispatcher; @@ -66,19 +66,22 @@ public abstract class MessageBase extends ContainerBase implements MessagePipe { */ public static EndpointConfiguration combineConfig(EndpointConfiguration defaults, EndpointConfiguration defined) { - EndpointConfiguration useDefaults = ofNullable(defaults).orElseGet( - EndpointConfiguration::new); + EndpointConfiguration useDefaults = ofNullable(defaults).orElseGet(EndpointConfiguration::new); return ifNotNullGet(defined, () -> mergeObject(deepCopy(useDefaults), defined)); } static Bundle extractBundle(String bundleString) { - return ifNotNullGet(bundleString, b -> fromString(Bundle.class, b)); + return ifNotNullGet(bundleString, b -> fromString(Bundle.class, b)); } static String normalizeNamespace(String configSpace) { return ofNullable(configSpace).orElse(DEFAULT_NAMESPACE); } + protected static String queueIdentifierStatic(BlockingQueue queue) { + return format("%08x", Objects.hash(queue)); + } + protected Bundle makeExceptionBundle(Envelope envelope, Exception exception) { Bundle bundle = new Bundle(envelope, exception); bundle.envelope.subType = SubType.EVENT; @@ -104,8 +107,8 @@ protected void pushQueueEntry(BlockingQueue queue, String stringBund } } - protected void receiveMessage(Envelope envelope, Map messageMap) { - receiveMessage(toStringMap(envelope), stringify(messageMap)); + protected String queueIdentifier() { + return queueIdentifierStatic(sourceQueue); } protected void receiveMessage(Map envelopeMap, Map messageMap) { @@ -125,7 +128,8 @@ protected void receiveMessage(Map attributesMap, String messageS final Envelope envelope; try { - envelope = convertToStrict(Envelope.class, attributesMap); + sanitizeAttributeMap(attributesMap); + envelope = convertTo(Envelope.class, attributesMap); } catch (Exception e) { attributesMap.put(INVALID_ENVELOPE_KEY, "true"); receiveException(attributesMap, messageString, e, null); @@ -142,6 +146,10 @@ protected void receiveMessage(Map attributesMap, String messageS } } + protected void receiveMessage(Envelope envelope, Map messageMap) { + receiveMessage(toStringMap(envelope), stringify(messageMap)); + } + protected void setSourceQueue(BlockingQueue queueForScope) { sourceQueue = queueForScope; } @@ -226,19 +234,6 @@ private void messageLoop(String id) { } } - private void shutdownExecutor() { - debug("Shutdown of %s", this); - executor.shutdown(); - } - - protected String queueIdentifier() { - return queueIdentifier(sourceQueue); - } - - protected static String queueIdentifier(BlockingQueue queue) { - return format("%08x", Objects.hash(queue)); - } - private void receiveBundle(Bundle bundle) { receiveBundle(stringify(bundle)); } @@ -259,6 +254,29 @@ private void receiveException(Map attributesMap, String messageS receiveBundle(stringify(bundle)); } + private void sanitizeAttributeMap(Map attributesMap) { + String subFolderRaw = attributesMap.get(SUBFOLDER_PROPERTY_KEY); + if (subFolderRaw != null) { + SubFolder subFolder = catchToElse(() -> SubFolder.fromValue(subFolderRaw), SubFolder.UNKNOWN); + if (!subFolder.value().equals(subFolderRaw)) { + attributesMap.put(SUBFOLDER_PROPERTY_KEY, subFolder.value()); + } + } + + String subTypeRaw = attributesMap.get(SUBTYPE_PROPERTY_KEY); + if (subTypeRaw != null) { + SubType subType = catchToElse(() -> SubType.fromValue(subTypeRaw), SubType.UNKNOWN); + if (!subType.value().equals(subTypeRaw)) { + attributesMap.put(SUBTYPE_PROPERTY_KEY, subType.value()); + } + } + } + + private void shutdownExecutor() { + debug("Shutdown of %s", this); + executor.shutdown(); + } + @Override public void activate(Consumer bundleConsumer) { dispatcher = bundleConsumer; diff --git a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipe.java b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipe.java index 23370cc9d4..45da361165 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipe.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipe.java @@ -1,6 +1,7 @@ package com.google.bos.udmi.service.messaging.impl; import static com.google.udmi.util.Common.DEVICE_ID_KEY; +import static com.google.udmi.util.Common.PUBLISH_TIME_KEY; import static com.google.udmi.util.GeneralUtils.decodeBase64; import static com.google.udmi.util.GeneralUtils.encodeBase64; import static com.google.udmi.util.GeneralUtils.ifNotNullGet; @@ -35,15 +36,6 @@ */ public class TraceMessagePipe extends MessageBase { - private static final String DEVICES_DIR_NAME = "devices"; - private static final Map FOLDER_HACKS = new HashMap<>(); - - static { - // Some egregious hacks for dealing with legacy corner-cases data streams. - FOLDER_HACKS.put("discover", "discovery"); - FOLDER_HACKS.put("", null); - } - private final Map traceCounts = new HashMap<>(); private File traceOutFile; @@ -62,33 +54,21 @@ public static MessagePipe fromConfig(EndpointConfiguration config) { private void consumeTrace(File file) { try { Map traceBundle = asMap(file); - Envelope envelope = makeEnvelope(traceBundle); + Map envelopeMap = makeEnvelope(traceBundle); Map message = asMap(decodeBase64((String) traceBundle.get("data"))); - receiveMessage(envelope, message); + receiveMessage(envelopeMap, message); } catch (Exception e) { e.printStackTrace(); } } - @Nullable - private SubFolder getBundleSubfolder(Map attributes) { - String subFolder = attributes.get("subFolder"); - return ifNotNullGet(FOLDER_HACKS.getOrDefault(subFolder, subFolder), SubFolder::fromValue); - } - - private Envelope makeEnvelope(Map bundle) { + private Map makeEnvelope(Map bundle) { try { @SuppressWarnings("unchecked") Map attributes = (Map) bundle.get("attributes"); - Envelope envelope = new Envelope(); - envelope.subFolder = getBundleSubfolder(attributes); - envelope.subType = ifNotNullGet(attributes.get("subType"), SubType::fromValue); - envelope.deviceId = attributes.get(DEVICE_ID_KEY); - envelope.projectId = attributes.get("projectId"); - envelope.deviceRegistryId = attributes.get("deviceRegistryId"); - envelope.publishTime = ifNotNullGet(ofNullable(attributes.get(Common.PUBLISH_TIME_KEY)) - .orElse((String) bundle.get("publish_time")), JsonUtil::getDate); - return envelope; + attributes.put(PUBLISH_TIME_KEY, + ofNullable(attributes.get(PUBLISH_TIME_KEY)).orElse((String) bundle.get("publish_time"))); + return attributes; } catch (Exception e) { throw new RuntimeException("While extracting envelope from bundle", e); } diff --git a/udmis/src/test/configs/bridge_pod.json b/udmis/src/test/configs/bridge_pod.json index 33ac501a75..dcb3d1492d 100644 --- a/udmis/src/test/configs/bridge_pod.json +++ b/udmis/src/test/configs/bridge_pod.json @@ -9,7 +9,7 @@ "recv_id": "from", "send_id": "to" }, - "to": { + "morf": { "recv_id": "mark", "send_id": "mend" } diff --git a/udmis/src/test/configs/trace_pod.json b/udmis/src/test/configs/trace_pod.json index f9d566a759..b3d1311438 100644 --- a/udmis/src/test/configs/trace_pod.json +++ b/udmis/src/test/configs/trace_pod.json @@ -16,7 +16,7 @@ "recv_id": "../tests/traces/simple/devices", "send_id": "out/simple.trace" }, - "to": { + "morf": { "recv_id": "mark", "send_id": "mend" } diff --git a/udmis/src/test/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipeTest.java b/udmis/src/test/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipeTest.java index 1aca542974..8a18b6117e 100644 --- a/udmis/src/test/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipeTest.java +++ b/udmis/src/test/java/com/google/bos/udmi/service/messaging/impl/TraceMessagePipeTest.java @@ -107,8 +107,7 @@ public void tracePlayback() { List errors = consumed.stream().filter(bundle -> bundle.envelope.subFolder == SubFolder.ERROR) - .map(bundle -> (String) bundle.message) - .collect(Collectors.toList()); + .map(bundle -> (String) bundle.message).toList(); assertEquals(0, errors.size(), "expected message errors"); Set devices = From c7a3c68fdf36bd11e920a9e8f1259a6c4dd98154 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Sun, 26 Nov 2023 07:50:14 -0400 Subject: [PATCH 14/30] Fix testing out --- etc/validator.out | 2 +- tests/schemas/envelope/errors1.out | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/validator.out b/etc/validator.out index 9f3488d4c8..3a35c73fbf 100644 --- a/etc/validator.out +++ b/etc/validator.out @@ -330,7 +330,7 @@ sites/udmi_site_model/out/devices/GAT-123/event_invalid.out "level" : 500 }, { "message" : "While converting to json node: 1 schema violations found", - "detail" : "event_invalid: While converting to json node: 1 schema violations found REDACTED_ERROR; 1 schema violations found; /subFolder: instance value (\"invalid\") not found in enum (possible values: [\"udmi\",\"update\",\"error\",\"cloud\",\"discovery\",\"mapping\",\"system\",\"gateway\",\"swarm\",\"localnet\",\"pointset\",\"validation\",\"blobset\",\"monitoring\"])", + "detail" : "event_invalid: While converting to json node: 1 schema violations found REDACTED_ERROR; 1 schema violations found; /subFolder: instance value (\"invalid\") not found in enum (possible values: [\"udmi\",\"update\",\"unknown\",\"error\",\"cloud\",\"discovery\",\"mapping\",\"system\",\"gateway\",\"swarm\",\"localnet\",\"pointset\",\"validation\",\"blobset\",\"monitoring\"])", "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 diff --git a/tests/schemas/envelope/errors1.out b/tests/schemas/envelope/errors1.out index 49244df1d2..47a57f1745 100644 --- a/tests/schemas/envelope/errors1.out +++ b/tests/schemas/envelope/errors1.out @@ -2,5 +2,5 @@ /deviceId: ECMA 262 regex "^[A-Z]{2,6}-[0-9]{1,6}$" does not match input string "fcu-1" /deviceNumId: ECMA 262 regex "^[0-9]+$" does not match input string "921302198324X" /deviceRegistryId: ECMA 262 regex "^[a-zA-Z][-a-zA-Z0-9._+~%]*[a-zA-Z0-9]$" does not match input string "test/registry" - /subType: instance value (5) not found in enum (possible values: ["event","command","config","state","query","reply","model"]) + /subType: instance value (5) not found in enum (possible values: ["unknown","event","command","config","state","query","reply","model"]) object has missing required properties (["projectId","subFolder"]) From e6332aaecda8abbe9017aca80e3ac6bed198c4b8 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Mon, 27 Nov 2023 05:55:14 -0500 Subject: [PATCH 15/30] Remove old project ref --- udmis/etc/k8s_udmis.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/udmis/etc/k8s_udmis.yaml b/udmis/etc/k8s_udmis.yaml index d00581bc3d..c108c930b3 100644 --- a/udmis/etc/k8s_udmis.yaml +++ b/udmis/etc/k8s_udmis.yaml @@ -57,8 +57,6 @@ spec: sources: - secret: name: clearblade.json - - secret: - name: dbo-rews-dev.json - name: tmp emptyDir: medium: Memory From ae9084b2423096f53db94e1ee04b4c361f5e07b7 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 06:50:21 -0500 Subject: [PATCH 16/30] Change unknown to invalid --- .gencode_hash.txt | 4 ++-- gencode/docs/monitoring.html | 4 ++-- gencode/java/udmi/schema/Envelope.java | 4 ++-- schema/envelope.json | 4 ++-- udmis/etc/k8s_config.yaml | 2 +- .../google/bos/udmi/service/messaging/impl/MessageBase.java | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.gencode_hash.txt b/.gencode_hash.txt index a414daad15..911623dfe4 100644 --- a/.gencode_hash.txt +++ b/.gencode_hash.txt @@ -13,7 +13,7 @@ d2fe30200f9a1c530d144137a7774b5b97f268aae41185bfcb25de274d399250 gencode/docs/e d39b9cbfbdd3f6b5a34396b6a7ab0e7f92a07264ce216d26e57483b22f751f37 gencode/docs/event_system.html 04a864b8b37b01a3b234ba86e7e123ebaf3c98bba497c2f06d23c4550ae8194e gencode/docs/event_validation.html fa237fe9d96c2809bf9562abdc5c4f7c7d933010beb5c54407caa4453dec30f9 gencode/docs/metadata.html -68ccd5121e5d25ef0502844ca9bb4ed67861e0ecf9180db3e2bd90ec54da4144 gencode/docs/monitoring.html +0aa459d46e8dc4c1f40e574cad107ae212d08b41fdde1660c052a4dd21a197cc gencode/docs/monitoring.html 180b32717db748e164a185b163ef9a97aa83d9d6add306283d5b9852d04af947 gencode/docs/persistent_device.html 5d039d607af9ec75ee552dfe36b16c702687ea16f5663f41fc49b4533b86e00d gencode/docs/properties.html 1766f84518a315fe57e4a4bf934c0a386ad61d87091754a6bab097c686c16019 gencode/docs/readme.md @@ -54,7 +54,7 @@ e809df42a73ed843977e447fa3333140dcdbe7d09b4e88bf1b8fd52fbaf9ae1a gencode/java/u b01b7a05f21651a01a605466bb80b2e537059dcbb41f33734c851432da57f657 gencode/java/udmi/schema/EndpointConfiguration.java dd2eb479a8e93a851c535c8b40fbd62e152bd60e0473f3b23800ec61f798bed0 gencode/java/udmi/schema/Entry.java 06758aca1e0043ddf343b504030f47bb19260e99a82e2d66f12e86092a2434ca gencode/java/udmi/schema/Enumerate.java -351ec1178006a546be1218eea6ea604f4b4160ec2db3a71c7095443d546ff9f5 gencode/java/udmi/schema/Envelope.java +8a51984458d96d1798d067005902defa57410167a27dcfb0b730ea38a3326011 gencode/java/udmi/schema/Envelope.java e9f5c77be81486b6b8c6d88f70f2d50583d8c3fafa2ac09ead80f44b8d5e751e gencode/java/udmi/schema/Event.java 9ff3d1819b5a92713b26f56d08c77dcbb467c6ffc10f75744c6b6a227a13319b gencode/java/udmi/schema/ExecutionConfiguration.java 2e77a29988e6c17875d3f781be931e3b835bb310639f60d3bb28f24405a86bed gencode/java/udmi/schema/FamilyDiscoveryConfig.java diff --git a/gencode/docs/monitoring.html b/gencode/docs/monitoring.html index 3086a54750..d7e8f363e8 100644 --- a/gencode/docs/monitoring.html +++ b/gencode/docs/monitoring.html @@ -3945,7 +3945,7 @@

subFolderType: enum (of string)

Must be one of:

-
  • "udmi"
  • "update"
  • "unknown"
  • "error"
  • "cloud"
  • "discovery"
  • "mapping"
  • "system"
  • "gateway"
  • "swarm"
  • "localnet"
  • "pointset"
  • "validation"
  • "blobset"
  • "monitoring"
+
  • "udmi"
  • "update"
  • "invalid"
  • "error"
  • "cloud"
  • "discovery"
  • "mapping"
  • "system"
  • "gateway"
  • "swarm"
  • "localnet"
  • "pointset"
  • "validation"
  • "blobset"
  • "monitoring"
@@ -3995,7 +3995,7 @@

subTypeType: enum (of string)

Must be one of:

-
  • "unknown"
  • "event"
  • "command"
  • "config"
  • "state"
  • "query"
  • "reply"
  • "model"
+
  • "invalid"
  • "event"
  • "command"
  • "config"
  • "state"
  • "query"
  • "reply"
  • "model"
diff --git a/gencode/java/udmi/schema/Envelope.java b/gencode/java/udmi/schema/Envelope.java index ef6b7b7df6..e96b011d64 100644 --- a/gencode/java/udmi/schema/Envelope.java +++ b/gencode/java/udmi/schema/Envelope.java @@ -116,7 +116,7 @@ public enum SubFolder { UDMI("udmi"), UPDATE("update"), - UNKNOWN("unknown"), + INVALID("invalid"), ERROR("error"), CLOUD("cloud"), DISCOVERY("discovery"), @@ -167,7 +167,7 @@ public static Envelope.SubFolder fromValue(String value) { @Generated("jsonschema2pojo") public enum SubType { - UNKNOWN("unknown"), + INVALID("invalid"), EVENT("event"), COMMAND("command"), CONFIG("config"), diff --git a/schema/envelope.json b/schema/envelope.json index 9a6427993c..c5a70378b6 100644 --- a/schema/envelope.json +++ b/schema/envelope.json @@ -46,7 +46,7 @@ "enum": [ "udmi", "update", - "unknown", + "invalid", "error", "cloud", "discovery", @@ -63,7 +63,7 @@ }, "subType": { "enum": [ - "unknown", + "invalid", "event", "command", "config", diff --git a/udmis/etc/k8s_config.yaml b/udmis/etc/k8s_config.yaml index d4008d1094..5edbe1ca9a 100644 --- a/udmis/etc/k8s_config.yaml +++ b/udmis/etc/k8s_config.yaml @@ -5,4 +5,4 @@ metadata: data: GCP_IOT_OPTIONS: disabled UDMI_NAMESPACE: @UDMI_NAMESPACE@ - PIPE_SHUNT: mapping_engine,essential-keep-197822,udmi-external + PIPE_SHUNT: mapping_engine,essential-keep-197822,dbo-rews-dev diff --git a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java index b93474d896..c8d60d13af 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java @@ -257,7 +257,7 @@ private void receiveException(Map attributesMap, String messageS private void sanitizeAttributeMap(Map attributesMap) { String subFolderRaw = attributesMap.get(SUBFOLDER_PROPERTY_KEY); if (subFolderRaw != null) { - SubFolder subFolder = catchToElse(() -> SubFolder.fromValue(subFolderRaw), SubFolder.UNKNOWN); + SubFolder subFolder = catchToElse(() -> SubFolder.fromValue(subFolderRaw), SubFolder.INVALID); if (!subFolder.value().equals(subFolderRaw)) { attributesMap.put(SUBFOLDER_PROPERTY_KEY, subFolder.value()); } @@ -265,7 +265,7 @@ private void sanitizeAttributeMap(Map attributesMap) { String subTypeRaw = attributesMap.get(SUBTYPE_PROPERTY_KEY); if (subTypeRaw != null) { - SubType subType = catchToElse(() -> SubType.fromValue(subTypeRaw), SubType.UNKNOWN); + SubType subType = catchToElse(() -> SubType.fromValue(subTypeRaw), SubType.INVALID); if (!subType.value().equals(subTypeRaw)) { attributesMap.put(SUBTYPE_PROPERTY_KEY, subType.value()); } From 5f1d73fdc08f8485708b4a7e335cc6ae362cc82d Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 07:07:06 -0500 Subject: [PATCH 17/30] Adding extra field --- .../src/main/java/daq/pubber/PointsetManager.java | 7 ++++++- pubber/src/main/java/daq/pubber/Pubber.java | 6 +----- pubber/src/main/java/daq/pubber/SystemManager.java | 13 +++++++++++-- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/pubber/src/main/java/daq/pubber/PointsetManager.java b/pubber/src/main/java/daq/pubber/PointsetManager.java index 823874724d..a2eff7712a 100644 --- a/pubber/src/main/java/daq/pubber/PointsetManager.java +++ b/pubber/src/main/java/daq/pubber/PointsetManager.java @@ -16,7 +16,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import daq.pubber.Pubber.ExtraPointsetEvent; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -27,6 +26,7 @@ import udmi.schema.PointPointsetModel; import udmi.schema.PointPointsetState; import udmi.schema.PointsetConfig; +import udmi.schema.PointsetEvent; import udmi.schema.PointsetModel; import udmi.schema.PointsetState; import udmi.schema.PubberConfiguration; @@ -262,4 +262,9 @@ private void sendDevicePoints() { host.publish(pointsetEvent); } + static class ExtraPointsetEvent extends PointsetEvent { + + // This extraField exists only to trigger schema parsing errors. + public Object extraField; + } } diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index 744ae0131d..c34a221812 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -44,6 +44,7 @@ import daq.pubber.MqttPublisher.InjectedMessage; import daq.pubber.MqttPublisher.InjectedState; import daq.pubber.MqttPublisher.PublisherException; +import daq.pubber.PointsetManager.ExtraPointsetEvent; import daq.pubber.PubSubClient.Bundle; import java.io.ByteArrayOutputStream; import java.io.File; @@ -1557,9 +1558,4 @@ public void error(String message, Throwable e) { deviceManager.localLog(message, Level.TRACE, getTimestamp(), stackTraceString(e)); } - static class ExtraPointsetEvent extends PointsetEvent { - - // This extraField exists only to trigger schema parsing errors. - public Object extraField; - } } diff --git a/pubber/src/main/java/daq/pubber/SystemManager.java b/pubber/src/main/java/daq/pubber/SystemManager.java index a3fc208b3a..2362160db3 100644 --- a/pubber/src/main/java/daq/pubber/SystemManager.java +++ b/pubber/src/main/java/daq/pubber/SystemManager.java @@ -3,6 +3,7 @@ import static com.google.udmi.util.GeneralUtils.catchOrElse; import static com.google.udmi.util.GeneralUtils.catchToNull; import static com.google.udmi.util.GeneralUtils.ifNotNullGet; +import static com.google.udmi.util.GeneralUtils.ifNotNullThen; import static com.google.udmi.util.GeneralUtils.ifNotTrueThen; import static com.google.udmi.util.GeneralUtils.isTrue; import static com.google.udmi.util.JsonUtil.getTimestamp; @@ -13,6 +14,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.udmi.util.CleanDateFormat; +import com.google.udmi.util.GeneralUtils; import java.io.File; import java.io.PrintStream; import java.time.Instant; @@ -31,6 +33,7 @@ import udmi.schema.Operation; import udmi.schema.Operation.SystemMode; import udmi.schema.PubberConfiguration; +import udmi.schema.State; import udmi.schema.StateSystemHardware; import udmi.schema.StateSystemOperation; import udmi.schema.SystemConfig; @@ -79,7 +82,7 @@ public class SystemManager extends ManagerBase { } private final List logentries = new ArrayList<>(); - private final SystemState systemState; + private final ExtraSystemState systemState; private final ManagerHost host; private int systemEventCount; private SystemConfig systemConfig; @@ -94,7 +97,7 @@ public SystemManager(ManagerHost host, PubberConfiguration configuration) { info("Device start time is " + getTimestamp(DEVICE_START_TIME)); - systemState = new SystemState(); + systemState = new ExtraSystemState(); systemState.operation = new StateSystemOperation(); if (!isTrue(options.noLastStart)) { @@ -108,6 +111,8 @@ public SystemManager(ManagerHost host, PubberConfiguration configuration) { systemState.serial_no = configuration.serialNo; systemState.last_config = new Date(0); + ifNotNullThen(options.extraField, value -> systemState.extra_field = value); + updateState(); } @@ -296,4 +301,8 @@ public void pubberLogMessage(String logMessage, Level level, String timestamp, S logEntry.detail = detail; publishLogMessage(logEntry); } + + class ExtraSystemState extends SystemState { + public String extra_field; + } } From 05f30555ae40278e66401ff627d3ee303e56f018 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 07:18:31 -0500 Subject: [PATCH 18/30] Fix simple --- pubber/src/main/java/daq/pubber/SystemManager.java | 6 ++---- tests/schemas/envelope/errors1.out | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pubber/src/main/java/daq/pubber/SystemManager.java b/pubber/src/main/java/daq/pubber/SystemManager.java index 2362160db3..66e6924051 100644 --- a/pubber/src/main/java/daq/pubber/SystemManager.java +++ b/pubber/src/main/java/daq/pubber/SystemManager.java @@ -14,7 +14,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.udmi.util.CleanDateFormat; -import com.google.udmi.util.GeneralUtils; import java.io.File; import java.io.PrintStream; import java.time.Instant; @@ -33,7 +32,6 @@ import udmi.schema.Operation; import udmi.schema.Operation.SystemMode; import udmi.schema.PubberConfiguration; -import udmi.schema.State; import udmi.schema.StateSystemHardware; import udmi.schema.StateSystemOperation; import udmi.schema.SystemConfig; @@ -111,7 +109,7 @@ public SystemManager(ManagerHost host, PubberConfiguration configuration) { systemState.serial_no = configuration.serialNo; systemState.last_config = new Date(0); - ifNotNullThen(options.extraField, value -> systemState.extra_field = value); + ifNotNullThen(options.extraField, value -> systemState.extraField = value); updateState(); } @@ -303,6 +301,6 @@ public void pubberLogMessage(String logMessage, Level level, String timestamp, S } class ExtraSystemState extends SystemState { - public String extra_field; + public String extraField; } } diff --git a/tests/schemas/envelope/errors1.out b/tests/schemas/envelope/errors1.out index 47a57f1745..aebd2e7bba 100644 --- a/tests/schemas/envelope/errors1.out +++ b/tests/schemas/envelope/errors1.out @@ -2,5 +2,5 @@ /deviceId: ECMA 262 regex "^[A-Z]{2,6}-[0-9]{1,6}$" does not match input string "fcu-1" /deviceNumId: ECMA 262 regex "^[0-9]+$" does not match input string "921302198324X" /deviceRegistryId: ECMA 262 regex "^[a-zA-Z][-a-zA-Z0-9._+~%]*[a-zA-Z0-9]$" does not match input string "test/registry" - /subType: instance value (5) not found in enum (possible values: ["unknown","event","command","config","state","query","reply","model"]) + /subType: instance value (5) not found in enum (possible values: ["invalid","event","command","config","state","query","reply","model"]) object has missing required properties (["projectId","subFolder"]) From 8930595b775c9a48d271dc34ab14a28d8c3e318c Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 07:45:26 -0500 Subject: [PATCH 19/30] Make shunt configuration a secret --- udmis/Dockerfile.udmis | 1 + udmis/bin/run | 11 +++++------ udmis/etc/k8s_config.yaml | 1 - udmis/etc/k8s_udmis.yaml | 9 +++------ 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/udmis/Dockerfile.udmis b/udmis/Dockerfile.udmis index 7f92620ddf..1e07e0554c 100644 --- a/udmis/Dockerfile.udmis +++ b/udmis/Dockerfile.udmis @@ -20,5 +20,6 @@ ADD bin/ bin/ ADD var/ var/ ENV CLEARBLADE_CONFIGURATION=/udmi/clearblade.json +ENV SHUNT_CONFIGURATION=/udmi/shunt-config.json CMD ["/root/bin/run", "var/prod_pod.json"] diff --git a/udmis/bin/run b/udmis/bin/run index 09c240fdad..3d2c15ff29 100755 --- a/udmis/bin/run +++ b/udmis/bin/run @@ -48,14 +48,13 @@ else echo No UDMI_NAMESPACE defined, so no UDMI_PREFIX fi -if [[ -n $PIPE_SHUNT ]]; then - readarray -td, a <<<"$PIPE_SHUNT,"; unset 'a[-1]'; declare -p a; - export SHUNT_NAME=${UDMI_PREFIX}$(echo ${a[0]}) - export SHUNT_FROM=$(echo ${a[1]}) - export SHUNT_MORF=$(echo ${a[2]}) +if [[ -n $SHUNT_CONFIGURATION && -s $SHUNT_CONFIGURATION ]]; then + export SHUNT_NAME=${UDMI_PREFIX}$(jq -r .name $SHUNT_CONFIGURATION) + export SHUNT_FROM=$(jq -r .from $SHUNT_CONFIGURATION) + export SHUNT_MORF=$(jq -r .morf $SHUNT_CONFIGURATION) echo Configured udmi shunt $SHUNT_NAME between $SHUNT_FROM and $SHUNT_MORF else - echo No PIPE_SHUNT defined, so no SHUNT_NAME configured. + echo No SHUNT_CONFIGURATION defined, so no SHUNT_NAME configured. fi if [[ -n $CLEARBLADE_PROJECT ]]; then diff --git a/udmis/etc/k8s_config.yaml b/udmis/etc/k8s_config.yaml index 5edbe1ca9a..c30141119e 100644 --- a/udmis/etc/k8s_config.yaml +++ b/udmis/etc/k8s_config.yaml @@ -5,4 +5,3 @@ metadata: data: GCP_IOT_OPTIONS: disabled UDMI_NAMESPACE: @UDMI_NAMESPACE@ - PIPE_SHUNT: mapping_engine,essential-keep-197822,dbo-rews-dev diff --git a/udmis/etc/k8s_udmis.yaml b/udmis/etc/k8s_udmis.yaml index c108c930b3..9cda16db66 100644 --- a/udmis/etc/k8s_udmis.yaml +++ b/udmis/etc/k8s_udmis.yaml @@ -45,18 +45,15 @@ spec: configMapKeyRef: name: config key: GCP_IOT_OPTIONS - - name: PIPE_SHUNT - valueFrom: - configMapKeyRef: - name: config - key: PIPE_SHUNT - optional: true volumes: - name: secret-volume projected: sources: - secret: name: clearblade.json + - secret: + name: shunt-config.json + optional: true - name: tmp emptyDir: medium: Memory From a90cf9973e781c0659aea375d08f233d1586cc6e Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 08:14:18 -0500 Subject: [PATCH 20/30] Clean up state handling to reflect raw message --- udmis/bin/run | 3 +-- .../bos/udmi/service/core/StateProcessor.java | 23 +++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/udmis/bin/run b/udmis/bin/run index 3d2c15ff29..71353a3d3f 100755 --- a/udmis/bin/run +++ b/udmis/bin/run @@ -80,6 +80,5 @@ env echo # Run with exec to replace shell so java receives SIGTERM signal. -java -jar $POD_JAR $POD_CONFIG || true +exec java -jar $POD_JAR $POD_CONFIG -tail -f /dev/null diff --git a/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java b/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java index 3ad4577866..acff81e0cb 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java @@ -1,7 +1,7 @@ package com.google.bos.udmi.service.core; import static com.google.udmi.util.GeneralUtils.ifNotNullThen; -import static com.google.udmi.util.JsonUtil.convertToStrict; +import static com.google.udmi.util.JsonUtil.convertTo; import static com.google.udmi.util.JsonUtil.stringify; import static com.google.udmi.util.JsonUtil.toMap; import static com.google.udmi.util.MessageUpgrader.STATE_SCHEMA; @@ -33,9 +33,17 @@ public class StateProcessor extends ProcessorBase { @Override protected void defaultHandler(Object originalMessage) { + MessageContinuation continuation = getContinuation(originalMessage); + Envelope envelope = continuation.getEnvelope(); + envelope.subType = SubType.STATE; + envelope.subFolder = UPDATE; + + reflectMessage(envelope, stringify(originalMessage)); + Object upgradedMessage = new MessageUpgrader(STATE_SCHEMA, originalMessage).upgrade(); - StateUpdate stateMessage = convertToStrict(StateUpdate.class, upgradedMessage); - shardStateUpdate(getContinuation(originalMessage), stateMessage); + StateUpdate stateMessage = convertTo(StateUpdate.class, upgradedMessage); + shardStateUpdate(continuation, envelope, stateMessage); + updateLastStart(getContinuation(originalMessage).getEnvelope(), stateMessage); } @@ -49,11 +57,7 @@ protected void registerHandlers() { registerHandler(StateUpdate.class, this::stateHandler); } - private void shardStateUpdate(MessageContinuation continuation, StateUpdate message) { - Envelope envelope = continuation.getEnvelope(); - envelope.subType = SubType.STATE; - envelope.subFolder = UPDATE; - reflectMessage(envelope, stringify(message)); + private void shardStateUpdate(MessageContinuation continuation, Envelope envelope, StateUpdate message) { continuation.publish(message); String originalTransaction = envelope.transactionId; AtomicInteger txnSuffix = new AtomicInteger(); @@ -80,7 +84,8 @@ private void shardStateUpdate(MessageContinuation continuation, StateUpdate mess } private void stateHandler(StateUpdate message) { - shardStateUpdate(getContinuation(message), message); + MessageContinuation continuation = getContinuation(message); + shardStateUpdate(continuation, continuation.getEnvelope(), message); } } From ecbfaeda6a356b4adf2d8902b43828b998022171 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 08:19:34 -0500 Subject: [PATCH 21/30] Add some debugging --- .../com/google/bos/udmi/service/messaging/impl/MessageBase.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java index c8d60d13af..da7e032125 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/messaging/impl/MessageBase.java @@ -259,6 +259,7 @@ private void sanitizeAttributeMap(Map attributesMap) { if (subFolderRaw != null) { SubFolder subFolder = catchToElse(() -> SubFolder.fromValue(subFolderRaw), SubFolder.INVALID); if (!subFolder.value().equals(subFolderRaw)) { + debug("Coerced subFolder " + subFolderRaw + " to " + subFolder.value()); attributesMap.put(SUBFOLDER_PROPERTY_KEY, subFolder.value()); } } @@ -266,6 +267,7 @@ private void sanitizeAttributeMap(Map attributesMap) { String subTypeRaw = attributesMap.get(SUBTYPE_PROPERTY_KEY); if (subTypeRaw != null) { SubType subType = catchToElse(() -> SubType.fromValue(subTypeRaw), SubType.INVALID); + debug("Coerced subFolder " + subTypeRaw + " to " + subType.value()); if (!subType.value().equals(subTypeRaw)) { attributesMap.put(SUBTYPE_PROPERTY_KEY, subType.value()); } From 40d2b7e749157e624cdd22a0f8f26bed50bdea6d Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 08:24:51 -0500 Subject: [PATCH 22/30] Fix validator out --- etc/validator.out | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/etc/validator.out b/etc/validator.out index 3a35c73fbf..d33a4f4cf0 100644 --- a/etc/validator.out +++ b/etc/validator.out @@ -102,9 +102,9 @@ sites/udmi_site_model/out/devices/AHU-1/state.out "sub_folder" : "update", "sub_type" : "state", "status" : { - "message" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold", - "detail" : "state_update: Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold REDACTED_ERROR", - "category" : "validation.device.schema", + "message" : "Multiple validation errors", + "detail" : "Timestamp jitter REDACTED_DURATION (REDACTED_TIMESTAMP to REDACTED_TIMESTAMP) exceeds REDACTED_DURATION threshold; While converting to json node: 1 schema violations found", + "category" : "validation.device.multiple", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 }, @@ -114,6 +114,12 @@ sites/udmi_site_model/out/devices/AHU-1/state.out "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 + }, { + "message" : "While converting to json node: 1 schema violations found", + "detail" : "state_update: While converting to json node: 1 schema violations found REDACTED_ERROR; 1 schema violations found; /system: object instance has properties which are not allowed by the schema: [\"extraField\"]", + "category" : "validation.device.schema", + "timestamp" : "REDACTED_TIMESTAMP", + "level" : 500 } ] } :::::::::::::: @@ -316,9 +322,9 @@ sites/udmi_site_model/out/devices/GAT-123/event_invalid.out "sub_folder" : "invalid", "sub_type" : "event", "status" : { - "message" : "Multiple validation errors", - "detail" : "Unknown schema subFolder 'event_invalid' for GAT-123; While converting to json node: 1 schema violations found", - "category" : "validation.device.multiple", + "message" : "Unknown schema subFolder 'event_invalid' for GAT-123", + "detail" : "event_invalid: Unknown schema subFolder 'event_invalid' for GAT-123 REDACTED_ERROR", + "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 }, @@ -328,12 +334,6 @@ sites/udmi_site_model/out/devices/GAT-123/event_invalid.out "category" : "validation.device.schema", "timestamp" : "REDACTED_TIMESTAMP", "level" : 500 - }, { - "message" : "While converting to json node: 1 schema violations found", - "detail" : "event_invalid: While converting to json node: 1 schema violations found REDACTED_ERROR; 1 schema violations found; /subFolder: instance value (\"invalid\") not found in enum (possible values: [\"udmi\",\"update\",\"unknown\",\"error\",\"cloud\",\"discovery\",\"mapping\",\"system\",\"gateway\",\"swarm\",\"localnet\",\"pointset\",\"validation\",\"blobset\",\"monitoring\"])", - "category" : "validation.device.schema", - "timestamp" : "REDACTED_TIMESTAMP", - "level" : 500 } ] } :::::::::::::: From 1f2868cd43a423bf343ae9c9d4c508bb098b3d50 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 08:43:12 -0500 Subject: [PATCH 23/30] Fix unit test --- .../com/google/bos/udmi/service/core/StateProcessor.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java b/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java index acff81e0cb..2d4e81d9e2 100644 --- a/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java +++ b/udmis/src/main/java/com/google/bos/udmi/service/core/StateProcessor.java @@ -57,7 +57,8 @@ protected void registerHandlers() { registerHandler(StateUpdate.class, this::stateHandler); } - private void shardStateUpdate(MessageContinuation continuation, Envelope envelope, StateUpdate message) { + private void shardStateUpdate(MessageContinuation continuation, Envelope envelope, + StateUpdate message) { continuation.publish(message); String originalTransaction = envelope.transactionId; AtomicInteger txnSuffix = new AtomicInteger(); @@ -85,7 +86,9 @@ private void shardStateUpdate(MessageContinuation continuation, Envelope envelop private void stateHandler(StateUpdate message) { MessageContinuation continuation = getContinuation(message); - shardStateUpdate(continuation, continuation.getEnvelope(), message); + Envelope envelope = continuation.getEnvelope(); + reflectMessage(envelope, stringify(message)); + shardStateUpdate(continuation, envelope, message); } } From 374f8add5e0c330f05ee2efd672302c30996b4da Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 10:34:42 -0500 Subject: [PATCH 24/30] Better schema out markings --- bin/test_itemized | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/test_itemized b/bin/test_itemized index 0106ccf4cb..24415d4253 100755 --- a/bin/test_itemized +++ b/bin/test_itemized @@ -117,9 +117,9 @@ while read -u 7 action test_name pubber_opts; do mv $SITE_PATH/out $SITE_PATH/out-$test_marker if [[ -s $SCHEMA_OUT ]]; then - cat $SCHEMA_OUT | sed -e "s/^/$test_name /" >> $SCHEMA_ITEMIZED + sed -e "s/^/$test_marker $test_name /" < $SCHEMA_OUT >> $SCHEMA_ITEMIZED else - echo $test_name >> $SCHEMA_ITEMIZED + echo $test_marker $test_name schema ok >> $SCHEMA_ITEMIZED fi done 7< $INPUT_FILE From c71d9d04265cd783f0618ec35f9a21fd962ee71d Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 10:38:35 -0500 Subject: [PATCH 25/30] Other extra system out --- pubber/src/main/java/daq/pubber/Pubber.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pubber/src/main/java/daq/pubber/Pubber.java b/pubber/src/main/java/daq/pubber/Pubber.java index c34a221812..ea9ddb7a4a 100644 --- a/pubber/src/main/java/daq/pubber/Pubber.java +++ b/pubber/src/main/java/daq/pubber/Pubber.java @@ -46,6 +46,7 @@ import daq.pubber.MqttPublisher.PublisherException; import daq.pubber.PointsetManager.ExtraPointsetEvent; import daq.pubber.PubSubClient.Bundle; +import daq.pubber.SystemManager.ExtraSystemState; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.OutputStream; @@ -136,7 +137,7 @@ public class Pubber extends ManagerBase implements ManagerHost { private static final ImmutableMap, String> MESSAGE_TOPIC_SUFFIX_MAP = new ImmutableMap.Builder, String>() .put(State.class, MqttDevice.STATE_TOPIC) - .put(SystemState.class, MqttDevice.STATE_TOPIC) // Used for badState option + .put(ExtraSystemState.class, MqttDevice.STATE_TOPIC) // Used for badState option .put(SystemEvent.class, getEventsSuffix("system")) .put(PointsetEvent.class, getEventsSuffix("pointset")) .put(ExtraPointsetEvent.class, getEventsSuffix("pointset")) From 04c88ba71516b29ab433ab63cbf74eb154d2ece7 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 11:02:24 -0500 Subject: [PATCH 26/30] Fix itemized schema out --- bin/test_itemcheck | 4 ++-- bin/test_itemized | 2 +- etc/schema_itemized.out | 53 +++++++++++++++++++++++++---------------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/bin/test_itemcheck b/bin/test_itemcheck index 02fdbf8059..7a1c8ee4e4 100755 --- a/bin/test_itemcheck +++ b/bin/test_itemcheck @@ -11,10 +11,10 @@ GOLDEN_SCHEMAS=$UDMI_ROOT/etc/schema_itemized.out failures= -echo Comparing diff $RESULTS_OUT $GOLDEN_FILE... +echo Comparing diff $RESULTS_OUT $GOLDEN_FILE diff -bu $RESULTS_OUT $GOLDEN_FILE || failures+="results " -echo Comparing diff $SCHEMA_OUT $GOLDEN_SCHEMAS... +echo Comparing diff $SCHEMA_OUT $GOLDEN_SCHEMAS diff -bu $SCHEMA_OUT $GOLDEN_SCHEMAS || failures+="schema " [[ -z $failures ]] || fail itemized checks failied because of $failures diff --git a/bin/test_itemized b/bin/test_itemized index 24415d4253..1a7a6fb550 100755 --- a/bin/test_itemized +++ b/bin/test_itemized @@ -119,7 +119,7 @@ while read -u 7 action test_name pubber_opts; do if [[ -s $SCHEMA_OUT ]]; then sed -e "s/^/$test_marker $test_name /" < $SCHEMA_OUT >> $SCHEMA_ITEMIZED else - echo $test_marker $test_name schema ok >> $SCHEMA_ITEMIZED + echo $test_marker $test_name >> $SCHEMA_ITEMIZED fi done 7< $INPUT_FILE diff --git a/etc/schema_itemized.out b/etc/schema_itemized.out index c430be893c..de48626c61 100644 --- a/etc/schema_itemized.out +++ b/etc/schema_itemized.out @@ -1,21 +1,32 @@ -broken_config -config_logging -device_config_acked -feature_enumeration -gateway_proxy_events -pointset_remove_point -pointset_request_extraneous -pointset_sample_rate -state_make_model -system_last_update -system_last_update RESULT fail schemas event_pointset_stable STABLE 5/5 Schema violations found -system_last_update RESULT pass schemas event_system_stable STABLE 5/5 All schema validations passed -system_last_update RESULT pass schemas state_update_stable STABLE 5/5 All schema validations passed -system_min_loglevel -system_mode_restart -too_much_state -valid_serial_no -valid_serial_no RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed -valid_serial_no RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed -valid_serial_no RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed -writeback_success +01 device_config_acked +02 pointset_request_extraneous +03 pointset_remove_point +04 pointset_remove_point +05 too_much_state +06 feature_enumeration +07 valid_serial_no RESULT pass schemas event_pointset_alpha ALPHA 5/5 All schema validations passed +07 valid_serial_no RESULT pass schemas event_system_alpha ALPHA 5/5 All schema validations passed +07 valid_serial_no RESULT pass schemas state_update_alpha ALPHA 5/5 All schema validations passed +08 writeback_success +09 writeback_success +10 pointset_sample_rate +11 system_mode_restart +12 config_logging +13 broken_config +14 broken_config +15 broken_config +16 broken_config +17 broken_config +18 system_last_update RESULT fail schemas event_pointset_stable STABLE 5/5 Schema violations found +18 system_last_update RESULT fail schemas state_update_stable STABLE 5/5 Schema violations found +18 system_last_update RESULT pass schemas event_system_stable STABLE 5/5 All schema validations passed +19 state_make_model +23 valid_serial_no +24 system_last_update +25 system_min_loglevel +26 system_min_loglevel +30 gateway_proxy_events +31 gateway_proxy_events +35 gateway_proxy_events +36 gateway_proxy_events +40 device_config_acked From 119ca9f872693eed98f5ae8baed784de700de596 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 11:35:54 -0500 Subject: [PATCH 27/30] Fix itemized out --- etc/test_itemized.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/test_itemized.out b/etc/test_itemized.out index b3e6813978..686d3ece5b 100644 --- a/etc/test_itemized.out +++ b/etc/test_itemized.out @@ -21,7 +21,7 @@ 17 CPBLTY fail system broken_config.logging ALPHA 0/1 Timeout after 30s waiting for log category `system.config.receive` level `DEBUG` to be logged 17 RESULT fail system broken_config ALPHA 0/5 expected: but was: 18 RESULT pass system system_last_update STABLE 5/5 Sequence complete -19 RESULT fail system state_make_model BETA 0/5 Pipeline type event error: While processing message REDACTED +19 RESULT fail system state_make_model BETA 0/5 Timeout waiting for no applicable system status 23 RESULT skip system valid_serial_no ALPHA 0/0 State testing disabled 24 RESULT fail system system_last_update STABLE 0/5 Timeout waiting for state last_config matches config timestamp 25 RESULT fail system system_min_loglevel ALPHA 0/5 Received state update with no-state device @@ -29,5 +29,5 @@ 30 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete 31 RESULT fail gateway gateway_proxy_events ALPHA 0/5 Timeout waiting for Missing data from AHU-22 35 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete -36 RESULT pass gateway gateway_proxy_events ALPHA 5/5 Sequence complete +36 RESULT fail gateway gateway_proxy_events ALPHA 0/5 Received state update with no-state device 40 RESULT skip system device_config_acked BETA 0/0 No config check for proxy device From 4e04a9d8e7639b3527a62f4969d29e3a7c3d3e3f Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 14:24:58 -0500 Subject: [PATCH 28/30] Minor cleanup of docker things --- udmis/Dockerfile.udmis | 8 +------- udmis/bin/run | 3 --- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/udmis/Dockerfile.udmis b/udmis/Dockerfile.udmis index 1e07e0554c..59fd40874f 100644 --- a/udmis/Dockerfile.udmis +++ b/udmis/Dockerfile.udmis @@ -2,13 +2,7 @@ FROM alpine:latest WORKDIR /root -RUN apk add openjdk17 bash gcompat curl jq python3 git - -RUN curl -sSL https://sdk.cloud.google.com | bash - -ENV PATH=$PATH:/root/google-cloud-sdk/bin - -RUN apk add mosquitto-clients curl jq +RUN apk add openjdk17 bash gcompat curl jq python3 # Workaround for https://github.com/grpc/grpc-java/issues/8751 ENV LD_PRELOAD=/lib/libgcompat.so.0 diff --git a/udmis/bin/run b/udmis/bin/run index 71353a3d3f..ea30a5400e 100755 --- a/udmis/bin/run +++ b/udmis/bin/run @@ -72,8 +72,6 @@ if nslookup metadata.google.internal; then echo Configured with GCP service account $GCP_SERVICE_ACCOUNT fi -gcloud config set account $GCP_SERVICE_ACCOUNT - echo echo Final java env: env @@ -81,4 +79,3 @@ echo # Run with exec to replace shell so java receives SIGTERM signal. exec java -jar $POD_JAR $POD_CONFIG - From e28b4a80957434461347d3afea904998183e64df Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 14:38:35 -0500 Subject: [PATCH 29/30] Make dockerfile like validator --- udmis/Dockerfile.udmis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/udmis/Dockerfile.udmis b/udmis/Dockerfile.udmis index 59fd40874f..dc1638ddd1 100644 --- a/udmis/Dockerfile.udmis +++ b/udmis/Dockerfile.udmis @@ -2,7 +2,7 @@ FROM alpine:latest WORKDIR /root -RUN apk add openjdk17 bash gcompat curl jq python3 +RUN apk add openjdk17 bash gcompat curl jq python3 git # Workaround for https://github.com/grpc/grpc-java/issues/8751 ENV LD_PRELOAD=/lib/libgcompat.so.0 From 7dfdc0f3056938daf1164bff56dbfff41afa3b87 Mon Sep 17 00:00:00 2001 From: Trevor Pering Date: Fri, 1 Dec 2023 19:15:25 -0500 Subject: [PATCH 30/30] Cleanup docker build --- udmis/Dockerfile.udmis | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/udmis/Dockerfile.udmis b/udmis/Dockerfile.udmis index dc1638ddd1..ca68e1455e 100644 --- a/udmis/Dockerfile.udmis +++ b/udmis/Dockerfile.udmis @@ -2,7 +2,7 @@ FROM alpine:latest WORKDIR /root -RUN apk add openjdk17 bash gcompat curl jq python3 git +RUN apk add bash openjdk17 gcompat curl jq # Workaround for https://github.com/grpc/grpc-java/issues/8751 ENV LD_PRELOAD=/lib/libgcompat.so.0