Merge pull request #123 from tsurai/controller_upgrade_fix

Fix controller upgrade process crash
hyperledger · Oct 13, 2023 · 6c360b8 · 6c360b8
2 parents 339d307 + 06ac333
commit 6c360b8
Show file tree

Hide file tree

Showing 8 changed files with 100 additions and 68 deletions.
diff --git a/.github/workflows/build-all.yml b/.github/workflows/build-all.yml
@@ -182,13 +182,16 @@ jobs:
         with:
           script: core.setFailed('${{ matrix.os_version }} - Not all nodes are in sync!')
       - name: Send node restart command
-        run: cd test && echo -e "indy-cli\npool create test gen_txn_file=/pool_transactions_genesis\npool connect test\nwallet create wallet key=1234\nwallet open wallet key=1234\ndid new seed=000000000000000000000000Trustee1\ndid use V4SGRU86Z58d6TV7PBUe6f\nledger pool-restart action=start" | docker-compose run indy-cli
+        id: node-restart
+        run: |
+          echo "::set-output name=restart-time::$(date +%s)"
+          cd test && echo -e "indy-cli\npool create test gen_txn_file=/pool_transactions_genesis\npool connect test\nwallet create wallet key=1234\nwallet open wallet key=1234\ndid new seed=000000000000000000000000Trustee1\ndid use V4SGRU86Z58d6TV7PBUe6f\nledger pool-restart action=start" | docker-compose run indy-cli
       - name: Get node restart status
         id: nodes_restarted
         run: |
-          sleep 20
-          OUTPUT="$(docker ps --filter "name=node" --format "{{.Status}}" | grep -o 'Up [0-9]\+ seconds' | cut -d' ' -f2 | awk -v ORS='\\n' '1' | head -c -2)"
-          echo "count=$({ echo -e $OUTPUT | wc -l ; echo -e $OUTPUT | awk -F= '$1<30' | wc -l; } | sort -n | head -1)" >> $GITHUB_OUTPUT
+          sleep 60
+          OUTPUT="$(docker container ls --filter "name=node" --format="{{.Names}}" | xargs -n1 docker container inspect --format="{{.State.StartedAt}}" | xargs -I {} date -d {} +%s)"
+          echo "::set-output name=count::$(echo "$OUTPUT" | awk -F= '$1>${{ steps.node-restart.outputs.restart-time }}' | wc -l)"
       - name: Fail if not all nodes restarted      
         if: steps.nodes_restarted.outputs.count != 4
         uses: actions/github-script@v3

diff --git a/build/init_and_run.sh b/build/init_and_run.sh
@@ -16,11 +16,13 @@ echo "INDY_NODE_IP=${INDY_NODE_IP:=0.0.0.0}"
 echo "INDY_NODE_PORT=${INDY_NODE_PORT:=9701}"
 echo "INDY_CLIENT_IP=${INDY_CLIENT_IP:=0.0.0.0}"
 echo "INDY_CLIENT_PORT=${INDY_CLIENT_PORT:=9702}"
+echo "CONTROLLER_CONTAINER_NAME=${CONTROLLER_CONTAINER_NAME:=indy_node_controller}"
 
 echo "INDY_NODE_SEED=[$(echo -n $INDY_NODE_SEED|wc -c) characters]"
 
 # Set NETWORK_NAME in indy_config.py
 awk '{if (index($1, "NETWORK_NAME") != 0) {print("NETWORK_NAME = \"'$INDY_NETWORK_NAME'\"")} else print($0)}' /etc/indy/indy_config.py> /tmp/indy_config.py
+sed -i -n -e '/^controlServiceHost=/!p' -e "\$acontrolServiceHost='$CONTROLLER_CONTAINER_NAME'" /tmp/indy_config.py
 mv /tmp/indy_config.py /etc/indy/indy_config.py
 
 # Init indy-node

diff --git a/controller/upgrade_indy.sh b/controller/upgrade_indy.sh
@@ -51,38 +51,54 @@ for container in $NODE_CONTAINER $CONTROLLER_CONTAINER; do
     fi
 
     # compare the old and new image hash to verify that a newer image has been pulled
-    if [[ "$($inspect_image '{{.Id}}')" != "$image_hash" && "$container" == "$NODE_CONTAINER" ]]; then
-        # parse and save various container parameter to supply to the new one
-        binds="-v $($inspect_container '{{join .HostConfig.Binds " -v "}}')"
-        ports="$($inspect_container '{{range $k, $v := .NetworkSettings.Ports}}{{range $v}}{{print "-p " .HostIp ":" .HostPort ":" $k " "}}{{end}}{{end}}' | sed 's/:::/[::]:/g')"
-        network="$($inspect_container '{{range $k, $v := .NetworkSettings.Networks}}{{$id := slice "'$($inspect_container {{.Id}})'" 0 12}}{{$aliases := join $v.Aliases " --network-alias "}}{{println "--network" $k "--ip" $v.IPAddress "--network-alias" $aliases "--network-alias" $id}}{{end}}' | head -n1)"
-        restart="--restart $($inspect_container '{{.HostConfig.RestartPolicy.Name}}')"
-        init="$($inspect_container '{{.HostConfig.Init}}' | sed 's/true/--init/g; /--init/!s/.*//')"
-        $inspect_container '{{join .Config.Env "\n"}}' > /tmp/envs
-        envs="$(if [ -s /tmp/envs ]; then echo "--env-file /tmp/envs"; else echo ""; fi)"
-        $inspect_container '{{range $k,$v := .Config.Labels}}{{printf "%s=%s\n" $k $v}}{{end}}' | grep -v "^org.opencontainers.image" > /tmp/labels
-        # patch image hash in the docker-compose labels
-        sed -i "s/sha256.*/$($inspect_image '{{.Id}}')/g" /tmp/labels
-        labels="$(if [ -s /tmp/labels ]; then echo "--label-file /tmp/labels"; else echo ""; fi)"
-
-        output=$($ENGINE container create --name ${container}_new $binds $ports $network $init $restart $envs $labels $image_name) 1> /dev/null
-        if [[ $? != 0 ]]; then
-            print_error "failed to create new node container: $output"
-            exit 1
+    if [[ "$($inspect_image '{{.Id}}')" == "$image_hash" ]]; then
+        echo "$container image is up to date"
+        continue
+    fi
+
+    # skip container recreation for the controller itself
+    if [[ "$container" == "$CONTROLLER_CONTAINER" ]]; then
+        echo "A new controller image has been pulled. Please recreate its container manually"
+        continue
+    fi
+
+    # parse and save various container parameter to supply to the new one
+    binds="-v $($inspect_container '{{join .HostConfig.Binds " -v "}}')"
+    ports="$($inspect_container '{{range $k, $v := .NetworkSettings.Ports}}{{range $v}}{{print "-p " .HostIp ":" .HostPort ":" $k " "}}{{end}}{{end}}' | sed 's/:::/[::]:/g')"
+    network="$($inspect_container '{{range $k, $v := .NetworkSettings.Networks}}{{$id := slice "'$($inspect_container {{.Id}})'" 0 12}}{{$aliases := join $v.Aliases " --network-alias "}}{{print " --network " $k " --network-alias " $aliases " --network-alias " $id}}{{with $v.IPAMConfig.IPv4Address}}{{print " --ip " .}}{{end}}{{end}}' | head -n1)"
+
+    # a missing network config might imply a shared internal network (network_mode:container)
+    if [[ -z "$network" ]]; then
+        if [[ "$($inspect_container '{{print " --network " .HostConfig.NetworkMode " "}}')" == container:* ]]; then
+            network=$($ENGINE container inspect $NODE_CONTAINER --format '{{print " --network container:" .Id')
         fi
+    fi
 
-        # exit bash script if any of the following fails
-        set -e
+    restart="--restart $($inspect_container '{{.HostConfig.RestartPolicy.Name}}')"
+    init="$($inspect_container '{{.HostConfig.Init}}' | sed 's/true/--init/g; /--init/!s/.*//')"
+    $inspect_container '{{join .Config.Env "\n"}}' > /tmp/envs
+    envs="$(if [ -s /tmp/envs ]; then echo "--env-file /tmp/envs"; else echo ""; fi)"
+    $inspect_container '{{range $k,$v := .Config.Labels}}{{printf "%s=%s\n" $k $v}}{{end}}' | grep -v "^org.opencontainers.image" > /tmp/labels
 
-        $ENGINE container stop $container 1> /dev/null
-        $ENGINE container rm $container 1> /dev/null
-        $ENGINE container rename ${container}_new $container 1> /dev/null
-        $ENGINE container start $container 1> /dev/null
+    # patch image hash in the docker-compose labels
+    sed -i "s/sha256.*/$($inspect_image '{{.Id}}')/g" /tmp/labels
+    labels="$(if [ -s /tmp/labels ]; then echo "--label-file /tmp/labels"; else echo ""; fi)"
 
-        rm /tmp/{envs,labels}
-    else
-        echo "$container image is up to date"
+    output=$($ENGINE container create --name ${container}_new $binds $ports $network $init $restart $envs $labels $image_name) 1> /dev/null
+    if [[ $? != 0 ]]; then
+        print_error "failed to create new container: $output"
+        exit 1
     fi
+
+    # exit bash script if any of the following fails
+    set -e
+
+    $ENGINE container stop $container 1> /dev/null
+    $ENGINE container rm $container 1> /dev/null
+    $ENGINE container rename ${container}_new $container 1> /dev/null
+    $ENGINE container start $container 1> /dev/null
+
+    rm /tmp/{envs,labels}
 done
 
 exit 0
diff --git a/run/.env b/run/.env
@@ -5,11 +5,10 @@ INDY_NODE_NAME=GS1Germany
 # Choose one of the avaiable images
 IMAGE=ghcr.io/hyperledger/indy-node-container/indy_node:latest-ubuntu18
 
-
 # You likely want to keep these default values:
 SOCK=/var/run/docker.sock
 NODE_CONTAINER_NAME=indy_node
-CONTROLLER_CONTAINER=indy_node_controller
+CONTROLLER_CONTAINER_NAME=indy_node_controller
 IMAGE_NAME_CONTROLLER=ghcr.io/hyperledger/indy-node-container/indy_node_controller:latest
 
 INDY_NODE_IP=0.0.0.0

diff --git a/run/docker-compose.yml b/run/docker-compose.yml
@@ -18,6 +18,7 @@ services:
       - INDY_CLIENT_PORT=${INDY_CLIENT_PORT}
       - INDY_NETWORK_NAME=${INDY_NETWORK_NAME}
       - INDY_NODE_NAME=${INDY_NODE_NAME}
+      - CONTROLLER_CONTAINER_NAME=${CONTROLLER_CONTAINER_NAME}
     logging:
         driver: "json-file"
         options:
@@ -30,11 +31,11 @@ services:
   indy-controller:
     image: ${IMAGE_NAME_CONTROLLER}
     init: true
-    container_name: ${CONTROLLER_CONTAINER}
+    container_name: ${CONTROLLER_CONTAINER_NAME}
     environment:
-     - NODE_CONTAINER=${NODE_CONTAINER_NAME}
-     - INDY_NETWORK_NAME=${INDY_NETWORK_NAME}
-     - CONTROLLER_CONTAINER=${CONTROLLER_CONTAINER}
+      - INDY_NETWORK_NAME=${INDY_NETWORK_NAME}
+      - NODE_CONTAINER=${NODE_CONTAINER_NAME}
+      - CONTROLLER_CONTAINER=${CONTROLLER_CONTAINER_NAME}
     volumes:
       - ./etc_indy:/etc/indy
       - ${SOCK}:/var/run/docker.sock
@@ -44,4 +45,5 @@ services:
           max-file: "5"
           max-size: "100m"      
     restart: always
-    network_mode: "service:indy-node"
+    depends_on:
+      - indy-node
diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml
@@ -2,7 +2,7 @@ version: "3.8"
 
 services:
   indy-node_1:
-    image: ${REFERENCE_IMAGE_NAME_NODE}
+    image: ${IMAGE_NAME_NODE}
     init: true
     container_name: node1
     ports:
@@ -12,11 +12,12 @@ services:
       - ./etc_indy/node1:/etc/indy
       - ./lib_indy:/var/lib/indy
     environment:
-     - INDY_NODE_NAME=Node1
-     - INDY_NODE_IP=${INDY_NODE_IP_1}
-     - INDY_NODE_PORT=${INDY_NODE_PORT_1}
-     - INDY_CLIENT_IP=${INDY_CLIENT_IP_1}
-     - INDY_CLIENT_PORT=${INDY_CLIENT_PORT_1}
+      - INDY_NODE_NAME=Node1
+      - INDY_NODE_IP=${INDY_NODE_IP_1}
+      - INDY_NODE_PORT=${INDY_NODE_PORT_1}
+      - INDY_CLIENT_IP=${INDY_CLIENT_IP_1}
+      - INDY_CLIENT_PORT=${INDY_CLIENT_PORT_1}
+      - CONTROLLER_CONTAINER_NAME=controller1
     env_file:
       - .env
     restart: always
@@ -25,7 +26,7 @@ services:
         ipv4_address: 10.133.133.1
 
   indy-node_2:
-    image: ${REFERENCE_IMAGE_NAME_NODE}
+    image: ${IMAGE_NAME_NODE}
     init: true
     container_name: node2
     ports:
@@ -35,11 +36,12 @@ services:
       - ./etc_indy/node2:/etc/indy
       - ./lib_indy:/var/lib/indy
     environment:
-     - INDY_NODE_NAME=Node2
-     - INDY_NODE_IP=${INDY_NODE_IP_2}
-     - INDY_NODE_PORT=${INDY_NODE_PORT_2}
-     - INDY_CLIENT_IP=${INDY_CLIENT_IP_2}
-     - INDY_CLIENT_PORT=${INDY_CLIENT_PORT_2}
+      - INDY_NODE_NAME=Node2
+      - INDY_NODE_IP=${INDY_NODE_IP_2}
+      - INDY_NODE_PORT=${INDY_NODE_PORT_2}
+      - INDY_CLIENT_IP=${INDY_CLIENT_IP_2}
+      - INDY_CLIENT_PORT=${INDY_CLIENT_PORT_2}
+      - CONTROLLER_CONTAINER_NAME=controller2
     env_file:
       - .env
     restart: always
@@ -48,7 +50,7 @@ services:
         ipv4_address: 10.133.133.2
 
   indy-node_3:
-    image: ${REFERENCE_IMAGE_NAME_NODE}
+    image: ${IMAGE_NAME_NODE}
     init: true
     container_name: node3
     ports:
@@ -58,11 +60,12 @@ services:
       - ./etc_indy/node3:/etc/indy
       - ./lib_indy:/var/lib/indy
     environment:
-     - INDY_NODE_NAME=Node3
-     - INDY_NODE_IP=${INDY_NODE_IP_3}
-     - INDY_NODE_PORT=${INDY_NODE_PORT_3}
-     - INDY_CLIENT_IP=${INDY_CLIENT_IP_3}
-     - INDY_CLIENT_PORT=${INDY_CLIENT_PORT_3}
+      - INDY_NODE_NAME=Node3
+      - INDY_NODE_IP=${INDY_NODE_IP_3}
+      - INDY_NODE_PORT=${INDY_NODE_PORT_3}
+      - INDY_CLIENT_IP=${INDY_CLIENT_IP_3}
+      - INDY_CLIENT_PORT=${INDY_CLIENT_PORT_3}
+      - CONTROLLER_CONTAINER_NAME=controller3
     env_file:
       - .env
     restart: always
@@ -81,11 +84,12 @@ services:
       - ./etc_indy/node4:/etc/indy
       - ./lib_indy:/var/lib/indy
     environment:
-     - INDY_NODE_NAME=Node4
-     - INDY_NODE_IP=${INDY_NODE_IP_4}
-     - INDY_NODE_PORT=${INDY_NODE_PORT_4}
-     - INDY_CLIENT_IP=${INDY_CLIENT_IP_4}
-     - INDY_CLIENT_PORT=${INDY_CLIENT_PORT_4}
+      - INDY_NODE_NAME=Node4
+      - INDY_NODE_IP=${INDY_NODE_IP_4}
+      - INDY_NODE_PORT=${INDY_NODE_PORT_4}
+      - INDY_CLIENT_IP=${INDY_CLIENT_IP_4}
+      - INDY_CLIENT_PORT=${INDY_CLIENT_PORT_4}
+      - CONTROLLER_CONTAINER_NAME=controller4
     env_file:
       - .env
     restart: always
@@ -143,10 +147,11 @@ services:
       - ./etc_indy:/etc/indy
       - $SOCK:/var/run/docker.sock
     restart: always
+    depends_on:
+      - indy-node_1
     networks:
       indy_node_container_test_network:
-        ipv4_address: 10.133.133.11
-
+        ipv4_address: 10.133.133.7
 
   indy-controller_2:
     image: ${IMAGE_NAME_CONTROLLER}
@@ -159,9 +164,11 @@ services:
       - ./etc_indy:/etc/indy
       - $SOCK:/var/run/docker.sock
     restart: always
+    depends_on:
+      - indy-node_2
     networks:
       indy_node_container_test_network:
-        ipv4_address: 10.133.133.12
+        ipv4_address: 10.133.133.8
 
   indy-controller_3:
     image: ${IMAGE_NAME_CONTROLLER}
@@ -174,9 +181,11 @@ services:
       - ./etc_indy:/etc/indy
       - $SOCK:/var/run/docker.sock
     restart: always
+    depends_on:
+      - indy-node_3
     networks:
       indy_node_container_test_network:
-        ipv4_address: 10.133.133.13
+        ipv4_address: 10.133.133.9
 
   indy-controller_4:
     image: ${IMAGE_NAME_CONTROLLER}
@@ -189,9 +198,11 @@ services:
       - ./etc_indy:/etc/indy
       - $SOCK:/var/run/docker.sock
     restart: always
+    depends_on:
+      - indy-node_4
     networks:
       indy_node_container_test_network:
-        ipv4_address: 10.133.133.14
+        ipv4_address: 10.133.133.10
 
 networks:
   indy_node_container_test_network:

diff --git a/test/etc_indy/indy_config.py b/test/etc_indy/indy_config.py
@@ -24,4 +24,4 @@
 NODE_INFO_DIR = '/var/lib/indy'
 
 logLevel = 0
-NETWORK_NAME = "local_test"
+NETWORK_NAME = 'local_test'
diff --git a/test/init-test-network.sh b/test/init-test-network.sh
@@ -19,5 +19,4 @@ docker run -v "${PWD}"/etc_indy:/etc/indy -v "${PWD}"/lib_indy:/var/lib/indy "$I
 for i in $(seq 1 $NODES); do
     mkdir -p "${PWD}"/etc_indy/node$i
     cp "${PWD}"/etc_indy/{indy.env,indy_config.py} "${PWD}"/etc_indy/node$i/
-    echo -e "\n# node controller container IP\ncontrolServiceHost = '10.133.133.1$i'" >> "${PWD}"/etc_indy/node$i/indy_config.py
 done