RHOSDT 3.4 disconnected (openshift#59562)

* RHOSDT 3.4 disconnected * fixes
yingzhanredhat · Dec 18, 2024 · 3516ed1 · 3516ed1
1 parent a1f641c
commit 3516ed1
Show file tree

Hide file tree

Showing 3 changed files with 113 additions and 10 deletions.
diff --git a/.../distributed-tracing-qe/openshift-distributed-tracing-qe-main__ocp-4.15-disconnected.yaml b/.../distributed-tracing-qe/openshift-distributed-tracing-qe-main__ocp-4.15-disconnected.yaml
@@ -41,9 +41,9 @@ tests:
         [
             {"name": "elasticsearch-operator", "source": "cs-redhat-operator-index", "channel": "stable-5.8", "install_namespace": "openshift-operators-redhat","target_namespaces": "", "operator_group": "openshift-operators-redhat"},
             {"name": "amq-streams", "source": "cs-redhat-operator-index", "channel": "stable", "install_namespace": "openshift-operators", "target_namespaces": ""},
-            {"name": "jaeger-product", "source": "cs-redhat-operator-index", "channel": "stable", "install_namespace": "openshift-distributed-tracing","target_namespaces": "", "operator_group": "openshift-distributed-tracing"},
-            {"name": "opentelemetry-product", "source": "cs-redhat-operator-index", "channel": "stable", "install_namespace": "openshift-opentelemetry-operator", "target_namespaces": "", "operator_group": "openshift-opentelemetry-operator"},
-            {"name": "tempo-product", "source": "cs-redhat-operator-index", "channel": "stable", "install_namespace": "openshift-tempo-operator", "target_namespaces": "", "operator_group": "openshift-tempo-operator"}
+            {"name": "jaeger-product", "source": "cs-jaeger", "channel": "stable", "install_namespace": "openshift-distributed-tracing","target_namespaces": "", "operator_group": "openshift-distributed-tracing"},
+            {"name": "opentelemetry-product", "source": "cs-otel", "channel": "stable", "install_namespace": "openshift-opentelemetry-operator", "target_namespaces": "", "operator_group": "openshift-opentelemetry-operator"},
+            {"name": "tempo-product", "source": "cs-tempo", "channel": "stable", "install_namespace": "openshift-tempo-operator", "target_namespaces": "", "operator_group": "openshift-tempo-operator"}
         ]
     test:
     - ref: distributed-tracing-install-disconnected

diff --git a/...ributed-tracing/install/disconnected/distributed-tracing-install-disconnected-commands.sh b/...ributed-tracing/install/disconnected/distributed-tracing-install-disconnected-commands.sh
@@ -113,17 +113,26 @@ storageConfig:
     path: /tmp/images
 mirror:
   operators:
-  - catalog: registry.stage.redhat.io/redhat/redhat-operator-index:v4.15
+  - catalog: brew.registry.redhat.io/rh-osbs/iib:875516
+    targetCatalog: rh-osbs/tempo
     packages:
     - name: tempo-product
       channels:
       - name: stable
+  - catalog: brew.registry.redhat.io/rh-osbs/iib:877598
+    targetCatalog: rh-osbs/otel
+    packages:
     - name: opentelemetry-product
       channels:
       - name: stable
+  - catalog: brew.registry.redhat.io/rh-osbs/iib:876810
+    targetCatalog: rh-osbs/jaeger
+    packages:
     - name: jaeger-product
       channels:
       - name: stable
+  - catalog: registry.stage.redhat.io/redhat/redhat-operator-index:v4.15
+    packages:
     - name: elasticsearch-operator
       channels:
       - name: stable-5.8
@@ -132,24 +141,117 @@ mirror:
       - name: stable
   additionalImages:
   # Used for running disconnected tests
-  - name: quay.io/minio/minio@sha256:7586fd4e11b48d7e41efdc6898b73147231da847cd0d4383d0a718ad2c98e6d8
-  - name: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen@sha256:a036b2ec403f445adc68bfcc359d5040a6603af4081a76a6cf02a5de55138a4e
-  - name: ghcr.io/grafana/tempo-operator/test-utils@sha256:7cf76bad82feb2cc338d23b29d7585c9399dfc8105bbab0c0636a776916aa3bc
+  - name: quay.io/minio/minio@sha256:dfc084fde47cbddece19c0616c280ea23c07f7d306dbbb135688c6ab60c6885c
+  - name: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest@sha256:d16e57246f71cb0b94390226bae23bb5b55d6a0fa5b0d067edf6e370a12e9799
+  - name: ghcr.io/grafana/tempo-operator/test-utils:main@sha256:abe652bdd34f9433b5bbe0e8100838b154f69b51aec0a9cc04fe381eef7bec84
   - name: docker.io/jaegertracing/vertx-create-span@sha256:6704312715644554fe4d51e0ce5cb0032e9231653ac61bdbdb5f290cb637d421
 EOF
 
     run_command "cd /tmp"
     run_command "curl -L -o oc-mirror.tar.gz https://mirror.openshift.com/pub/openshift-v4/amd64/clients/ocp/latest/oc-mirror.tar.gz && tar -xvzf oc-mirror.tar.gz && chmod +x oc-mirror"
     run_command "./oc-mirror --config=/tmp/image-set.yaml docker://${MIRROR_REGISTRY_HOST} --continue-on-error --ignore-history --source-skip-tls --dest-skip-tls || true"
     run_command "cp oc-mirror-workspace/results-*/mapping.txt ."
-    run_command "sed 's/registry.redhat.io/registry.stage.redhat.io/g' mapping.txt > mapping-stage.txt"
+    run_command "sed -e 's|registry.redhat.io|registry.stage.redhat.io|g' -e 's|brew.registry.stage.redhat.io/rh-osbs/tempo|brew.registry.redhat.io/rh-osbs/iib|g' -e 's|brew.registry.stage.redhat.io/rh-osbs/otel|brew.registry.redhat.io/rh-osbs/iib|g' -e 's|brew.registry.stage.redhat.io/rh-osbs/jaeger|brew.registry.redhat.io/rh-osbs/iib|g' mapping.txt > mapping-stage.txt"
     run_command "oc image mirror -a ${REG_CREDS} -f mapping-stage.txt --insecure --filter-by-os='.*'"
 
     # print and apply generated ICSP and catalog source
     run_command "cat oc-mirror-workspace/results-*/imageContentSourcePolicy.yaml"
     run_command "cat oc-mirror-workspace/results-*/catalogSource*"
     run_command "oc apply -f ./oc-mirror-workspace/results-*/"
 
+    CATALOG_SOURCE="cs-tempo"
+    local -i counter=0
+    local status=""
+    while [ $counter -lt 600 ]; do
+      counter+=20
+      echo "waiting ${counter}s"
+      sleep 20
+      status=$(oc -n openshift-marketplace get catalogsource "$CATALOG_SOURCE" -o=jsonpath="{.status.connectionState.lastObservedState}")
+      [[ $status = "READY" ]] && {
+        echo "$CATALOG_SOURCE Tempo CatalogSource created successfully"
+        break
+      }
+    done
+    [[ $status != "READY" ]] && {
+      echo "!!! fail to create Tempo CatalogSource"
+      run "oc get pods -o wide -n openshift-marketplace"
+      run "oc -n openshift-marketplace get catalogsource $CATALOG_SOURCE -o yaml"
+      run "oc -n openshift-marketplace get pods -l olm.catalogSource=$CATALOG_SOURCE -o yaml"
+      node_name=$(oc -n openshift-marketplace get pods -l olm.catalogSource="$CATALOG_SOURCE" -o=jsonpath='{.items[0].spec.nodeName}')
+      run "oc create ns debug-qe -o yaml | oc label -f - security.openshift.io/scc.podSecurityLabelSync=false \
+        pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/audit=privileged pod-security.kubernetes.io/warn=privileged --overwrite"
+      run "oc -n debug-qe debug node/$node_name -- chroot /host podman pull --authfile /var/lib/kubelet/config.json registry.stage.redhat.io/redhat/redhat-operator-index:v4.15"
+
+      run "oc get mcp,node"
+      run "oc get mcp worker -o yaml"
+      run "oc get mc $(oc get mcp/worker --no-headers | awk '{print $2}') -o=jsonpath={.spec.config.storage.files}|jq '.[] | select(.path==\"/var/lib/kubelet/config.json\")'"
+
+      return 1
+    }
+    return 0
+
+    CATALOG_SOURCE="cs-otel"
+    local -i counter=0
+    local status=""
+    while [ $counter -lt 600 ]; do
+      counter+=20
+      echo "waiting ${counter}s"
+      sleep 20
+      status=$(oc -n openshift-marketplace get catalogsource "$CATALOG_SOURCE" -o=jsonpath="{.status.connectionState.lastObservedState}")
+      [[ $status = "READY" ]] && {
+        echo "$CATALOG_SOURCE OTEL CatalogSource created successfully"
+        break
+      }
+    done
+    [[ $status != "READY" ]] && {
+      echo "!!! fail to create OTEL CatalogSource"
+      run "oc get pods -o wide -n openshift-marketplace"
+      run "oc -n openshift-marketplace get catalogsource $CATALOG_SOURCE -o yaml"
+      run "oc -n openshift-marketplace get pods -l olm.catalogSource=$CATALOG_SOURCE -o yaml"
+      node_name=$(oc -n openshift-marketplace get pods -l olm.catalogSource="$CATALOG_SOURCE" -o=jsonpath='{.items[0].spec.nodeName}')
+      run "oc create ns debug-qe -o yaml | oc label -f - security.openshift.io/scc.podSecurityLabelSync=false \
+        pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/audit=privileged pod-security.kubernetes.io/warn=privileged --overwrite"
+      run "oc -n debug-qe debug node/$node_name -- chroot /host podman pull --authfile /var/lib/kubelet/config.json registry.stage.redhat.io/redhat/redhat-operator-index:v4.15"
+
+      run "oc get mcp,node"
+      run "oc get mcp worker -o yaml"
+      run "oc get mc $(oc get mcp/worker --no-headers | awk '{print $2}') -o=jsonpath={.spec.config.storage.files}|jq '.[] | select(.path==\"/var/lib/kubelet/config.json\")'"
+
+      return 1
+    }
+    return 0
+
+    CATALOG_SOURCE="cs-jaeger"
+    local -i counter=0
+    local status=""
+    while [ $counter -lt 600 ]; do
+      counter+=20
+      echo "waiting ${counter}s"
+      sleep 20
+      status=$(oc -n openshift-marketplace get catalogsource "$CATALOG_SOURCE" -o=jsonpath="{.status.connectionState.lastObservedState}")
+      [[ $status = "READY" ]] && {
+        echo "$CATALOG_SOURCE Jaeger CatalogSource created successfully"
+        break
+      }
+    done
+    [[ $status != "READY" ]] && {
+      echo "!!! fail to create Jaeger CatalogSource"
+      run "oc get pods -o wide -n openshift-marketplace"
+      run "oc -n openshift-marketplace get catalogsource $CATALOG_SOURCE -o yaml"
+      run "oc -n openshift-marketplace get pods -l olm.catalogSource=$CATALOG_SOURCE -o yaml"
+      node_name=$(oc -n openshift-marketplace get pods -l olm.catalogSource="$CATALOG_SOURCE" -o=jsonpath='{.items[0].spec.nodeName}')
+      run "oc create ns debug-qe -o yaml | oc label -f - security.openshift.io/scc.podSecurityLabelSync=false \
+        pod-security.kubernetes.io/enforce=privileged pod-security.kubernetes.io/audit=privileged pod-security.kubernetes.io/warn=privileged --overwrite"
+      run "oc -n debug-qe debug node/$node_name -- chroot /host podman pull --authfile /var/lib/kubelet/config.json registry.stage.redhat.io/redhat/redhat-operator-index:v4.15"
+
+      run "oc get mcp,node"
+      run "oc get mcp worker -o yaml"
+      run "oc get mc $(oc get mcp/worker --no-headers | awk '{print $2}') -o=jsonpath={.spec.config.storage.files}|jq '.[] | select(.path==\"/var/lib/kubelet/config.json\")'"
+
+      return 1
+    }
+    return 0
+
     CATALOG_SOURCE="cs-redhat-operator-index"
     local -i counter=0
     local status=""
@@ -159,12 +261,12 @@ EOF
       sleep 20
       status=$(oc -n openshift-marketplace get catalogsource "$CATALOG_SOURCE" -o=jsonpath="{.status.connectionState.lastObservedState}")
       [[ $status = "READY" ]] && {
-        echo "$CATALOG_SOURCE CatalogSource created successfully"
+        echo "$CATALOG_SOURCE Stage CatalogSource created successfully"
         break
       }
     done
     [[ $status != "READY" ]] && {
-      echo "!!! fail to create QE CatalogSource"
+      echo "!!! fail to create Stage CatalogSource"
       run "oc get pods -o wide -n openshift-marketplace"
       run "oc -n openshift-marketplace get catalogsource $CATALOG_SOURCE -o yaml"
       run "oc -n openshift-marketplace get pods -l olm.catalogSource=$CATALOG_SOURCE -o yaml"

diff --git a/...istributed-tracing/install/disconnected/distributed-tracing-install-disconnected-ref.yaml b/...istributed-tracing/install/disconnected/distributed-tracing-install-disconnected-ref.yaml
@@ -1,6 +1,7 @@
 ref:
   as: distributed-tracing-install-disconnected
   from: upi-installer
+  timeout: 3h0m0s
   cli: latest
   grace_period: 10m
   commands: distributed-tracing-install-disconnected-commands.sh