Add noble_oemscript device-connector and updated docs. Updated image-…

…deploy.sh script

device-connectors/src/testflinger_device_connectors/data/muxpi/oemscript/image-deploy.sh

@@ -0,0 +1,181 @@
+#!/bin/bash
+
+exec 2>&1
+set -euox pipefail
+# This script was adapted to testflinger agent environment and used
+# to provision OEM devices with Ubuntu Noble images
+# Downloading ISO is not supported, because agent is in charge of ISO download
+
+usage()
+{
+cat <<EOF
+Usage:
+    $0 [OPTIONS] <TARGET_IP 1> <TARGET_IP 2> ...
+Options:
+    -h|--help        The manual of the script
+    --iso            ISO file path to be deployed on the target
+    -u|--user        The user of the target, default ubuntu
+    -o|--timeout     The timeout for doing the deployment, default 3600 seconds
+EOF
+}
+
+if [ $# -lt 3 ]; then
+    usage
+    exit
+fi
+
+TARGET_USER="ubuntu"
+TARGET_IPS=()
+ISO_PATH=
+ISO=
+STORE_PART=""
+TIMEOUT=3600
+SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+SSH="ssh $SSH_OPTS"
+SCP="scp $SSH_OPTS"
+
+if [ ! -d "$HOME/.cache/oem-scripts" ]; then
+    mkdir -p "$HOME/.cache/oem-scripts"
+fi
+
+OPTS="$(getopt -o u:o:l: --long iso:,user:,timeout:,local-config: -n 'image-deploy.sh' -- "$@")"
+eval set -- "${OPTS}"
+while :; do
+    case "$1" in
+        ('-h'|'--help')
+            usage
+	    exit;;
+        ('--iso')
+            ISO_PATH="$2"
+	    ISO=$(basename "$ISO_PATH")
+            shift 2;;
+        ('-u'|'--user')
+            TARGET_USER="$2"
+	    shift 2;;
+        ('-o'|'--timeout')
+            TIMEOUT="$2"
+            shift 2;;
+        ('-l'|'--local-config')
+            CONFIG_REPO_PATH="$2"
+            SKIP_GIT="TRUE"
+            shift 2;;
+	('--') shift; break ;;
+	(*) break ;;
+    esac
+done
+
+if [ ! -f "$ISO_PATH" ]; then
+    echo "No designated ISO file"
+    exit
+fi
+
+read -ra TARGET_IPS <<< "$@"
+
+for addr in "${TARGET_IPS[@]}";
+do
+    # Clear the knonw host
+    if [ -f "$HOME/.ssh/known_hosts" ]; then
+        ssh-keygen -f "$HOME/.ssh/known_hosts" -R "$addr"
+    fi
+
+    # Find the partitions
+    while read -r name fstype mountpoint;
+    do
+        echo "$name,$fstype,$mountpoint"
+        if [ "$fstype" = "ext4" ]; then
+            if [ "$mountpoint" = "/home/$TARGET_USER" ] || [ "$mountpoint" = "/" ]; then
+                STORE_PART="/dev/$name"
+                break
+            fi
+        fi
+    done < <($SSH "$TARGET_USER"@"$addr" -- lsblk -n -l -o NAME,FSTYPE,MOUNTPOINT)
+
+    if [ -z "$STORE_PART" ]; then
+        echo "Can't find partition to store ISO on target $addr"
+        exit
+    fi
+    RESET_PART="${STORE_PART:0:-1}2"
+    RESET_PARTUUID=$($SSH "$TARGET_USER"@"$addr" -- lsblk -n -o PARTUUID "$RESET_PART")
+    EFI_PART="${STORE_PART:0:-1}1"
+
+    # Copy ISO to the target
+    $SCP "$ISO_PATH" "$TARGET_USER"@"$addr":/home/"$TARGET_USER"
+
+    # Copy cloud-config redeploy to the target
+    $SSH "$TARGET_USER"@"$addr" -- mkdir -p /home/"$TARGET_USER"/redeploy/cloud-configs/redeploy
+    $SSH "$TARGET_USER"@"$addr" -- mkdir -p /home/"$TARGET_USER"/redeploy/cloud-configs/grub
+    $SCP "$CONFIG_REPO_PATH"/alloem-init/cloud-configs/redeploy/meta-data "$TARGET_USER"@"$addr":/home/"$TARGET_USER"/redeploy/cloud-configs/redeploy/
+    $SCP "$CONFIG_REPO_PATH"/alloem-init/cloud-configs/redeploy/user-data "$TARGET_USER"@"$addr":/home/"$TARGET_USER"/redeploy/cloud-configs/redeploy/
+    $SCP "$CONFIG_REPO_PATH"/alloem-init/cloud-configs/grub/redeploy.cfg "$TARGET_USER"@"$addr":/home/"$TARGET_USER"/redeploy/cloud-configs/grub/redeploy.cfg
+
+    # Copy ssh key from alloem-init injections to the target
+    $SCP -r "$CONFIG_REPO_PATH"/injections/alloem-init/chroot/minimal.standard.live.hotfix.squashfs/etc/ssh "$TARGET_USER"@"$addr":/home/"$TARGET_USER"/redeploy/ssh-config
+
+    # Umount the partitions
+    MOUNT=$($SSH "$TARGET_USER"@"$addr" -- lsblk -n -o MOUNTPOINT "$RESET_PART")
+    if [ -n "$MOUNT" ]; then
+        $SSH "$TARGET_USER"@"$addr" -- sudo umount "$RESET_PART"
+    fi
+    MOUNT=$($SSH "$TARGET_USER"@"$addr" -- lsblk -n -o MOUNTPOINT "$EFI_PART")
+    if [ -n "$MOUNT" ]; then
+        $SSH "$TARGET_USER"@"$addr" -- sudo umount "$EFI_PART"
+    fi
+
+    # Format partitions
+    $SSH "$TARGET_USER"@"$addr" -- sudo mkfs.vfat "$RESET_PART"
+    $SSH "$TARGET_USER"@"$addr" -- sudo mkfs.vfat "$EFI_PART"
+
+    # Mount ISO and reset partition
+    $SSH "$TARGET_USER"@"$addr" -- mkdir -p /home/"$TARGET_USER"/iso || true
+    $SSH "$TARGET_USER"@"$addr" -- mkdir -p /home/"$TARGET_USER"/reset || true
+    $SSH "$TARGET_USER"@"$addr" -- sudo mount -o loop /home/"$TARGET_USER"/"$ISO" /home/"$TARGET_USER"/iso || true
+    $SSH "$TARGET_USER"@"$addr" -- sudo mount "$RESET_PART" /home/"$TARGET_USER"/reset || true
+
+    # Sync ISO to the reset partition
+    $SSH "$TARGET_USER"@"$addr" -- sudo rsync -avP /home/"$TARGET_USER"/iso/ /home/"$TARGET_USER"/reset || true
+
+    # Sync cloud-configs to the reset partition
+    $SSH "$TARGET_USER"@"$addr" -- sudo mkdir -p /home/"$TARGET_USER"/reset/cloud-configs || true
+    $SSH "$TARGET_USER"@"$addr" -- sudo cp -r /home/"$TARGET_USER"/redeploy/cloud-configs/redeploy/ /home/"$TARGET_USER"/reset/cloud-configs/
+    $SSH "$TARGET_USER"@"$addr" -- sudo cp -r /home/"$TARGET_USER"/redeploy/ssh-config/ /home/"$TARGET_USER"/reset/
+    $SSH "$TARGET_USER"@"$addr" -- sudo cp /home/"$TARGET_USER"/redeploy/cloud-configs/grub/redeploy.cfg /home/"$TARGET_USER"/reset/boot/grub/grub.cfg
+    $SSH "$TARGET_USER"@"$addr" -- sudo sed -i "s/RP_PARTUUID/${RESET_PARTUUID}/" /home/"$TARGET_USER"/reset/boot/grub/grub.cfg
+
+    # Reboot the target
+    $SSH "$TARGET_USER"@"$addr" -- sudo reboot || true
+done
+
+# Clear the known hosts
+for addr in "${TARGET_IPS[@]}";
+do
+    if [ -f "$HOME/.ssh/known_hosts" ]; then
+        ssh-keygen -f "$HOME/.ssh/known_hosts" -R "$addr"
+    fi
+done
+
+# Polling the targets
+STARTED=("${TARGET_IPS[@]}")
+finished=0
+startTime=$(date +%s)
+while :;
+do
+    sleep 180
+    currentTime=$(date +%s)
+    if [[ $((currentTime - startTime)) -gt $TIMEOUT ]]; then
+        echo "Timeout is reached, deployment are not finished"
+        break
+    fi
+
+    for addr in "${STARTED[@]}";
+    do
+        if $SSH "$TARGET_USER"@"$addr" -- exit; then
+            STARTED=("${STARTED[@]/$addr}")
+            finished=$((finished + 1))
+        fi
+    done
+
+    if [ $finished -eq ${#TARGET_IPS[@]} ]; then
+        echo "Deployment are done"
+        break
+    fi
+done

device-connectors/src/testflinger_device_connectors/devices/__init__.py

@@ -41,6 +41,7 @@
     "fake_connector",
     "hp_oemscript",
     "lenovo_oemscript",
+    "noble_oemscript",
     "maas2",
     "multi",
     "muxpi",

device-connectors/src/testflinger_device_connectors/devices/noble_oemscript/__init__.py

@@ -0,0 +1,44 @@
+# Copyright (C) 2023 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Ubuntu OEM Recovery Provisioning for OEM laptops with Ubuntu Noble series.
+Starting from 24.04 OEM uses the same image and script for all vendors.
+Use this for systems that can use the oem image-deploy.sh script
+for provisioning.
+"""
+
+import logging
+
+from testflinger_device_connectors.devices import (
+    DefaultDevice,
+    RecoveryError,
+    catch,
+)
+from .noble_oemscript import NobleOemScript
+
+logger = logging.getLogger(__name__)
+
+
+class DeviceConnector(DefaultDevice):
+    """Tool for provisioning Noble OEM devices with an oem image."""
+
+    @catch(RecoveryError, 46)
+    def provision(self, args):
+        """Method called when the command is invoked."""
+        device = NobleOemScript(args.config, args.job_data)
+        logger.info("BEGIN provision")
+        logger.info("Provisioning device")
+        device.provision()
+        logger.info("END provision")

device-connectors/src/testflinger_device_connectors/devices/noble_oemscript/noble_oemscript.py

@@ -0,0 +1,28 @@
+# Copyright (C) 2023 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Ubuntu OEM Script provisioning for OEM devices with Ubuntu Noble series
+For systems that use the oem image-deploy.sh script for provisioning
+"""
+
+import logging
+from testflinger_device_connectors.devices.oemscript.oemscript import OemScript
+
+logger = logging.getLogger(__name__)
+
+
+class NobleOemScript(OemScript):
+    """Device Agent for Noble OEM devices."""
+
+    distro = "noble"

device-connectors/src/testflinger_device_connectors/devices/oemscript/oemscript.py

@@ -29,6 +29,7 @@
 )
 
 logger = logging.getLogger(__name__)
+attachments_dir = os.path.join(Path.cwd(), "attachments/provision")
 
 
 class OemScript:
@@ -90,43 +91,57 @@ def provision(self):
 
         provision_data = self.job_data.get("provision_data", {})
         image_url = provision_data.get("url")
+        distro = self.distro
 
         # Download the .iso image from image_url
         if not image_url:
             logger.error(
                 "Please provide an image 'url' in the provision_data section"
             )
             raise ProvisioningError("No image url provided")
+
         try:
             image_file = download(image_url)
-
-            self.run_recovery_script(image_file)
+            self.run_recovery_script(image_file, distro)
 
             self.check_device_booted()
         finally:
             # remove the .iso image
             if image_file:
                 os.unlink(image_file)
 
-    def run_recovery_script(self, image_file):
+    def run_recovery_script(self, image_file, distro):
         """Download and run the OEM recovery script"""
         device_ip = self.config["device_ip"]
 
         data_path = Path(__file__).parent / "../../data/muxpi/oemscript"
-        recovery_script = data_path / "recovery-from-iso.sh"
-
         # Run the recovery script
         logger.info("Running recovery script")
-        cmd = [
-            recovery_script,
-            *self.extra_script_args,
-            "--local-iso",
-            image_file,
-            "--inject-ssh-key",
-            os.path.expanduser("~/.ssh/id_rsa.pub"),
-            "-t",
-            device_ip,
-        ]
+
+        if distro == "noble":
+            recovery_script = data_path / "image-deploy.sh"
+            cmd = [
+                recovery_script,
+                *self.extra_script_args,
+                "--iso",
+                image_file,
+                "--local-config",
+                attachments_dir,
+                device_ip,
+            ]
+        else:
+            recovery_script = data_path / "recovery-from-iso.sh"
+            cmd = [
+                recovery_script,
+                *self.extra_script_args,
+                "--local-iso",
+                image_file,
+                "--inject-ssh-key",
+                os.path.expanduser("~/.ssh/id_rsa.pub"),
+                "-t",
+                device_ip,
+            ]
+
         proc = subprocess.run(
             cmd,
             timeout=60 * 60,  # 1 hour - just in case

docs/reference/device-connector-types.rst

@@ -26,6 +26,8 @@ To specify the commands to run by the device in each test phase, set the ``testf
      - General device connector that does not support provisioning, but can run tests on a device where provisioning is not needed or not possible to do automatically.
    * - ``oemrecovery`` 
      - device connector where provisioning involves triggering a “recovery” mode to reset the image back to its original state.  This is useful for things like Ubuntu Core images with full disk encryption, which can be preloaded with cloud-init data to ensure user creation, then a command is configured for the device connector that will cause it to be reset back to its original state.
+   * - ``noble_oemscript``
+     - device connector to provision OEM Noble 24.04 image on HP/Dell/Lenovo. Uses image-deploy.sh script for provisioning.
    * - ``dell_oemscript``
      - This device connector is used for Dell OEM devices running certain versions of OEM supported images that can use a recovery partition to recover not only the same image, but in some cases, other OEM image versions as well.
    * - ``lenovo_oemscript`` 
@@ -263,6 +265,21 @@ The ``hp_oemscript`` device connector does not support any ``provision_data`` ke
        the ``zstd`` tool is supported) and
        flashed to the device, which will be used to boot up the DUT.
 
+nole_oemscript
+------------
+
+The ``noble_oemscript`` device connector supports the following ``provision_data`` keys.
+
+.. list-table:: Supported ``provision_data`` keys for ``noble_oemscript``
+   :header-rows: 1
+
+   * - Key
+     - Description
+   * - ``url``
+     - URL to the .iso image file which will be used to provision the device.
+   * - ``attachments``
+     - configuration files used for the system installation (user-data, meta-data, authorized_keys, etc.)
+
 .. _zapper_kvm:
 
 zapper_kvm