Merge pull request #315 from rh-ecosystem-edge/ut

Makefile

@@ -178,4 +178,4 @@ clean:
 	oc delete managedcluster $(EDGE_NAME); \
 	oc delete ns $(EDGE_NAME); \
 	oc rollout restart -n openshift-machine-api deployment/metal3; \
-	kcli delete vm $(EDGE_NAME)-m0 $(EDGE_NAME)-m1 $(EDGE_NAME)-m2 $(EDGE_NAME)-w0
+	kcli delete vm -y $(EDGE_NAME)-m0 $(EDGE_NAME)-m1 $(EDGE_NAME)-m2 $(EDGE_NAME)-w0

deploy-acm/deploy.sh

@@ -21,38 +21,30 @@ if ./verify.sh; then
 
     echo ">>>> Deploy manifests to install ACM ${OC_ACM_VERSION}"
     echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
-    oc apply -f 01-namespace.yml -o yaml --dry-run=client | oc apply -f -
+    oc apply -f 01-namespace.yml
     sleep 2
     oc apply -f 02-operatorgroup.yml
     sleep 2
     oc apply -f 03-subscription.yml
-    sleep 240
-
-    echo ">>>> Deploy ACM cr manifest"
-    echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>"
+    sleep 40
+    InstallPlan=$(oc --kubeconfig=${KUBECONFIG_HUB} get installplan -n open-cluster-management -o name)
+    RESOURCE_KIND=${InstallPlan%%/*}
+    RESOURCE_NAME=${InstallPlan##*/}
+    check_resource "${RESOURCE_KIND}" "${RESOURCE_NAME}" "Installed" "open-cluster-management" "${KUBECONFIG_HUB}"
+
+    echo ">>>> Deploy RHACM cr manifest"
+    echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
     oc apply -f 04-acm-cr.yml
+    # We need to give some time to ACM Operator to create the proper helmcharts
     sleep 60
 
-    echo ">>>> Wait until acm ready"
-    echo ">>>>>>>>>>>>>>>>>>>>>>>>>"
-    timeout=0
-    ready=false
-    sleep 240
-    while [ "${timeout}" -lt "120" ]; do
-        if [[ $(oc get pod -n open-cluster-management | grep -i running | wc -l) -eq $(oc get pod -n open-cluster-management | grep -v NAME | wc -l) ]]; then
-            ready=true
-            break
-        fi
-        sleep 5
-        timeout=$((timeout + 1))
+    echo ">>>> Wait until RHACM ready"
+    echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>"
+    for helmchart in $(oc --kubeconfig=${KUBECONFIG_HUB} get helmreleases -o name); do
+        RESOURCE_KIND=${helmchart%%/*}
+        RESOURCE_NAME=${helmchart##*/}
+        check_resource "${RESOURCE_KIND}" "${RESOURCE_NAME}" "Deployed" "open-cluster-management" "${KUBECONFIG_HUB}"
     done
-    if [ "$ready" == "false" ]; then
-        echo "timeout waiting for ACM pods "
-        exit 1
-    fi
-elif [[ $? -eq 50 ]]; then
-    echo ">>>> Verify failed...Some pods are failing..." #TODO change to remove and launch again
-    exit 50
 else
     echo ">>>> This step is not neccesary, everything looks ready"
 fi

deploy-disconnected-registry/deploy.sh

@@ -244,8 +244,10 @@ if [[ ${1} == 'hub' ]]; then
         deploy_registry 'hub'
         trust_internal_registry 'hub'
         check_resource "deployment" "${REGISTRY}" "Available" "${REGISTRY}" "${KUBECONFIG_HUB}"
-        render_file manifests/machine-config-certs.yaml 'hub'
         check_mcp 'hub'
+        render_file manifests/machine-config-certs-master.yaml 'hub'
+        render_file manifests/machine-config-certs-worker.yaml 'hub'
+        check_resource "mcp" "master" "Updated" "default" "${KUBECONFIG_HUB}"
         check_resource "deployment" "${REGISTRY}" "Available" "${REGISTRY}" "${KUBECONFIG_HUB}"
     else
         echo ">>>> This step to deploy registry on Hub is not neccesary, everything looks ready"

deploy-disconnected-registry/manifests/machine-config-certs-worker.yaml

@@ -0,0 +1,17 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  labels:
+    machineconfiguration.openshift.io/role: worker
+  name: update-localregistry-ca-certs
+spec:
+  config:
+    ignition:
+      version: 3.1.0
+    storage:
+      files:
+      - contents:
+          source: data:text/plain;charset=us-ascii;base64,$CA_CERT_DATA
+        filesystem: root
+        mode: 493
+        path: $PATH_CA_CERT

deploy-edgecluster/render_edgeclusters.sh

@@ -308,9 +308,10 @@ EOF
         echo ">> Checking Ignored Interfaces"
         echo "Edge-cluster: ${cluster}"
         echo "Master: ${master}"
-        IGN_IFACES=$(yq eval ".edgeclusters[${edgeclusternumber}].${cluster}.master${master}.ignore_ifaces" ${EDGECLUSTERS_FILE})
+        IGN_IFACES=$(yq eval ".edgeclusters[${edgeclusternumber}].[].master${master}.ignore_ifaces" ${EDGECLUSTERS_FILE})
         if [[ ${IGN_IFACES} != "null" ]]; then
-            yq eval -ojson ".edgeclusters[${edgeclusternumber}].${cluster}.master${master}.ignore_ifaces" ${EDGECLUSTERS_FILE} | jq -c '.[]' | while read IFACE; do
+            for IFACE in $(echo ${IGN_IFACES}); do
+
                 echo "Ignoring Interface: ${IFACE}"
                 echo "     - name: ${IFACE}" >>${OUTPUT}
 
@@ -325,7 +326,7 @@ EOF
          next-hop-interface: $CHANGE_EDGE_MASTER_PUB_INT
 EOF
         if [[ ${IGN_IFACES} != "null" ]]; then
-            yq eval -ojson ".edgeclusters[${edgeclusternumber}].${cluster}.master${master}.ignore_ifaces" ${EDGECLUSTERS_FILE} | jq -c '.[]' | while read IFACE; do
+            for IFACE in $(echo ${IGN_IFACES}); do
                 echo "Ignoring route for: ${IFACE}"
                 echo "       - next-hop-interface: ${IFACE}" >>${OUTPUT}
                 echo "         state: absent" >>${OUTPUT}

deploy-edgecluster/verify.sh

@@ -21,7 +21,7 @@ function check_aci() {
             break
         fi
         echo ">> Waiting for ACI"
-        echo "Edge-cluster: ${cluster}"
+        echo "Edgecluster: ${cluster}"
         echo "Current: $(oc --kubeconfig=${KUBECONFIG_HUB} get aci -n ${cluster} ${cluster} -o jsonpath='{.status.debugInfo.stateInfo}')"
         echo "Desired State: Cluster is Installed"
         echo

deploy-odf/deploy.sh

@@ -7,7 +7,7 @@ set -m
 
 function render_file() {
     SOURCE_FILE=${1}
-    if [[ $# -lt 1 ]]; then
+    if [[ ${#} -lt 1 ]]; then
         echo "Usage :"
         echo "  $0 <SOURCE FILE> <(optional) DESTINATION_FILE>"
         exit 1
@@ -25,10 +25,10 @@ function extract_vars() {
     # Extract variables from config file
     DISKS_PATH=${1}
     raw_disks=$(yq eval "${DISKS_PATH}" "${EDGECLUSTERS_FILE}" | sed s/null//)
-    disks=$(echo ${raw_disks} | tr -d '\ ' | sed s#-#,#g | sed 's/,*//' | sed 's/,*//')
+    disks=$(echo ${raw_disks} | tr -d '\ ' | sed 's/-/,/g' | sed 's/,*//' | sed 's/,*//')
     disks_count=$(echo ${disks} | sed 's/,/\n/g' | wc -l)
 
-    for node in $(oc --kubeconfig=${EDGE_KUBECONFIG} get nodes -o name | sed s#node\/##); do
+    for node in $(oc --kubeconfig=${EDGE_KUBECONFIG} get nodes -o name | cut -f2 -d/); do
         nodes+="${node},"
     done
 
@@ -102,49 +102,24 @@ if ! ./verify.sh; then
         oc --kubeconfig=${EDGE_KUBECONFIG} apply -f manifests/02-LSO-OperatorGroup.yaml
         sleep 2
         oc --kubeconfig=${EDGE_KUBECONFIG} apply -f manifests/03-LSO-Subscription.yaml
-        sleep 2
+        sleep 5
 
         echo ">>>> Waiting for subscription and crd on: ${edgecluster}"
         echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
-        timeout=0
-        ready=false
-        while [ "$timeout" -lt "1000" ]; do
-            echo KUBEEDGE=${EDGE_KUBECONFIG}
-            if [[ $(oc --kubeconfig=${EDGE_KUBECONFIG} get crd | grep localvolumes.local.storage.openshift.io | wc -l) -eq 1 ]]; then
-                ready=true
-                break
-            fi
-            echo "Waiting for CRD localvolumes.local.storage.openshift.io to be created"
-            sleep 5
-            timeout=$((timeout + 5))
+        declare -a LSOCRDS=("localvolumediscoveries.local.storage.openshift.io" "localvolumediscoveryresults.local.storage.openshift.io" "localvolumes.local.storage.openshift.io" "localvolumesets.local.storage.openshift.io")
+        for crd in ${LSOCRDS[@]}
+        do
+            check_resource "crd" "${crd}" "Established" "openshift-local-storage" "${EDGE_KUBECONFIG}"
         done
-        if [ "$ready" == "false" ]; then
-            echo timeout waiting for CRD localvolumes.local.storage.openshift.io
-            exit 1
-        fi
 
         echo ">>>> Render and apply manifests for LocalVolume on: ${edgecluster}"
         echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
         render_file manifests/04-LSO-LocalVolume.yaml
         sleep 20
 
-        echo ">>>> Waiting for: LSO PVs on ${edgecluster}"
+        echo ">>>> Waiting for: LSO LocalVolume on ${edgecluster}"
         echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
-        timeout=0
-        ready=false
-        while [ "$timeout" -lt "1000" ]; do
-            if [[ $(oc --kubeconfig=${EDGE_KUBECONFIG} get pv -o name | wc -l) -ge 3 ]]; then
-                ready=true
-                break
-            fi
-            sleep 5
-            timeout=$((timeout + 5))
-        done
-
-        if [ "$ready" == "false" ]; then
-            echo "timeout waiting for LSO PVs..."
-            exit 1
-        fi
+        check_resource "localvolume" "localstorage-disks-block" "Available" "openshift-local-storage" "${EDGE_KUBECONFIG}"
 
         echo ">>>> Deploy manifests to install ODF $OC_ODF_VERSION"
         echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"

deploy-odf/manifests/04-ODF-StorageCluster.yaml

@@ -25,5 +25,5 @@ spec:
     name: ocs-deviceset
     placement: {}
     portable: False
-    replica: 1
+    replica: 3
     resources: {}

deploy-worker/render_worker.sh

@@ -132,35 +132,11 @@ EOF
 
         echo ">>>> Deploying BMH Worker ${worker} for ${1}"
         oc --kubeconfig=${KUBECONFIG_HUB} apply -f ${OUTPUT}
+        # Give some time for the agent to be created
+        sleep 60
     done
 }
 
-function verify_worker() {
-
-    cluster=${1}
-    timeout=0
-    ready=false
-
-    echo ">>>> Waiting for Worker Agent: ${cluster}"
-    echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
-    while [ "$timeout" -lt "1000" ]; do
-        WORKER_AGENT=$(oc --kubeconfig=${KUBECONFIG_HUB} get agent -n ${cluster} --no-headers | grep worker | cut -f1 -d\ )
-        echo "Waiting for Worker's agent installation for edgecluster: ${cluster} - Agent: ${WORKER_AGENT}"
-        if [[ $(oc --kubeconfig=${KUBECONFIG_HUB} get agent -n ${cluster} ${WORKER_AGENT} -o jsonpath='{.status.conditions[?(@.reason=="InstallationCompleted")].status}') == True ]]; then
-            ready=true
-            break
-        fi
-        sleep 5
-        timeout=$((timeout + 1))
-    done
-
-    if [ "$ready" == "false" ]; then
-        echo "Timeout waiting for Worker's agent installation for edgecluster: ${cluster}"
-        exit 1
-    fi
-
-}
-
 # Load common vars
 source ${WORKDIR}/shared-utils/common.sh
 
@@ -180,6 +156,13 @@ index=0
 
 for EDGE in ${ALLEDGECLUSTERS}; do
     create_worker_definitions ${EDGE} ${index}
-    verify_worker ${EDGE}
+    WORKER_AGENT=$(oc --kubeconfig=${KUBECONFIG_HUB} get agent -n ${EDGE} --no-headers | grep worker | awk '{print $1}')
+    if [ "x${WORKER_AGENT}" == "x" ]; then
+        echo "Worker agent is empty for ${EDGE}, sleeping a bit more... 120 secs"
+        sleep 120
+        WORKER_AGENT=$(oc --kubeconfig=${KUBECONFIG_HUB} get agent -n ${EDGE} --no-headers | grep worker | awk '{print $1}')
+        echo "Now worker agent is ${WORKER_AGENT}"
+    fi
+    check_resource "agent" "${WORKER_AGENT}" "Installed" "${EDGE}" "${KUBECONFIG_HUB}"
     index=$((index + 1))
 done

finish-deployment/deploy.sh

@@ -4,38 +4,6 @@ set -o pipefail
 set -o nounset
 set -m
 
-function check_managedcluster() {
-    cluster=${1}
-    wait_time=${2}
-    condition=${3}
-    desired_status=${4}
-
-    timeout=0
-    ready=false
-
-    while [ "${timeout}" -lt "${wait_time}" ]; do
-        if [[ $(oc --kubeconfig=${KUBECONFIG_HUB} get managedcluster ${cluster} -o jsonpath="{.status.conditions[?(@.type==\"${condition}\")].status}") == "${desired_status}" ]]; then
-            ready=true
-            break
-        fi
-        echo ">> Waiting for ManagedCluster"
-        echo "Edge-cluster: ${cluster}"
-        echo "Condition: ${condition}"
-        echo "Desired State: ${desired_status}"
-        echo
-        timeout=$((timeout + 10))
-        sleep 10
-    done
-
-    if [ "$ready" == "false" ]; then
-        echo "Timeout waiting for Edge-cluster ${cluster} on condition ${condition}"
-        echo "Expected: ${desired_status} Current: $(oc --kubeconfig=${KUBECONFIG_HUB} get managedcluster ${cluster} -o jsonpath="{.status.conditions[?(@.type==\"${condition}\")].status}")"
-        exit 1
-    else
-        echo "ManagedCluster for ${cluster} condition: ${condition} verified"
-    fi
-}
-
 function detach_cluster() {
     # Function to clean cluster from hub
     cluster=${1}
@@ -143,11 +111,11 @@ function check_cluster() {
     cluster=${1}
     wait_time=10000
 
-    echo ">>>> Check edgecluster cluster: ${cluster}"
-    echo ">> Check ManagedCluster for edgecluster: ${cluster}"
-    check_managedcluster "${cluster}" "${wait_time}" "ManagedClusterConditionAvailable" "True"
-    check_managedcluster "${cluster}" "${wait_time}" "ManagedClusterImportSucceeded" "True"
-    check_managedcluster "${cluster}" "${wait_time}" "ManagedClusterJoined" "True"
+    echo ">>>> Check Edge cluster: ${cluster}"
+    echo ">> Check ManagedCluster for Edge: ${cluster}"
+    check_resource "managedcluster" "${cluster}" "ManagedClusterConditionAvailable" "edgecluster-deployer" "${KUBECONFIG_HUB}"
+    check_resource "managedcluster" "${cluster}" "ManagedClusterImportSucceeded" "edgecluster-deployer" "${KUBECONFIG_HUB}"
+    check_resource "managedcluster" "${cluster}" "ManagedClusterJoined" "edgecluster-deployer" "${KUBECONFIG_HUB}"
     echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
 }
 

hack/deploy-hub-local/ansible/tasks/base/01-packages.yaml

@@ -84,5 +84,7 @@
     - spice-protocol
     - qemu-kvm-ui-spice
     - golang
+    - glibc-langpack-es
+    - mosh
   ignore_errors: true
   tags: [ 01-packages ]

hack/deploy-hub-local/ansible/tasks/services/sshd/00-mosh-base.yaml

@@ -0,0 +1,8 @@
+---
+- name: MOSH Firewall rule
+  firewalld:
+    service: mosh
+    permanent: true
+    state: enabled
+    immediate: yes
+  tags: configure-sshd

hack/deploy-hub-local/bootstrap-ci.sh

@@ -40,7 +40,7 @@ function check_resource() {
     echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
     timeout=0
     ready=false
-    while [ "$timeout" -lt "1000" ]; do
+    while [ "${timeout}" -lt "1000" ]; do
         if [[ $(oc --kubeconfig=${KUBECONFIG_HUB} -n ${NAMESPACE} get ${RESOURCE} ${RESOURCE_NAME} -o jsonpath="{.status.conditions[?(@.type==\"${TYPE_STATUS}\")].status}") == 'True' ]]; then
             ready=true
             break

pipelines/bootstrap.sh

@@ -80,7 +80,7 @@ function check_resource() {
     echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
     timeout=0
     ready=false
-    while [ "$timeout" -lt "1000" ]; do
+    while [ "${timeout}" -lt "1000" ]; do
         if [[ $(oc --kubeconfig=${KUBECONFIG_HUB} -n ${NAMESPACE} get ${RESOURCE} ${RESOURCE_NAME} -o jsonpath="{.status.conditions[?(@.type==\"${TYPE_STATUS}\")].status}") == 'True' ]]; then
             ready=true
             break

shared-utils/common.sh

@@ -46,7 +46,7 @@ function check_resource() {
     echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
     timeout=0
     ready=false
-    while [ "$timeout" -lt "1000" ]; do
+    while [ "${timeout}" -lt "1000" ]; do
         if [[ $(oc --kubeconfig=${KUBE} -n ${NAMESPACE} get ${RESOURCE} ${RESOURCE_NAME} -o jsonpath="{.status.conditions[?(@.type==\"${TYPE_STATUS}\")].status}") == 'True' ]]; then
             ready=true
             break