Default user data for oem autoinstall (#377)

* device_connectors: Add default-user-data to oem_autoinstall

* Udpate docs for oem_autoinstall

* 1. Move data dir for oem_autoinstall 2. Add ssh copy id by device connector

* Increase the timeout for device status polling after provision

* 1.Update default-user-data 2.Fix default test user/password

.github/CODEOWNERS

@@ -1,4 +1,4 @@
 # OEM Software Engineering
 device-connectors/src/testflinger_device_connectors/devices/zapper_iot @canonical/oem-swe-iot
 device-connectors/src/testflinger_device_connectors/devices/oem_autoinstall @canonical/oem-swe-x86
-device-connectors/src/testflinger_device_connectors/data/muxpi/oem_autoinstall/provision-image.sh @canonical/oem-swe-x86
+device-connectors/src/testflinger_device_connectors/data/oem_autoinstall @canonical/oem-swe-x86

device-connectors/src/testflinger_device_connectors/data/oem_autoinstall/default-user-data

@@ -0,0 +1,86 @@
+#cloud-config
+# vim: syntax=yaml
+
+autoinstall:
+  version: 1
+
+  storage:
+    layout:
+      name: direct
+      match:
+        install-media: true
+
+  early-commands:
+    - "nmcli networking off"
+
+  late-commands:
+    - "bash /cdrom/sideloads/hook.sh late-commands"
+    - "mount -o rw,remount /cdrom"
+
+    # Rename factory reset EFI directory so firmware won't show it. Ignore
+    # errors on the way.
+    - "mv /cdrom/EFI /cdrom/efi.factory || true"
+    - "sed -i 's#/EFI/boot/bootx64.efi#/efi.factory/boot/bootx64.efi#g' /target/etc/grub.d/99_reset || true"
+    - "echo 'GRUB_CMDLINE_LINUX_DEFAULT=\"quiet splash nomodeset modprobe.blacklist=nouveau nouveau.modeset=0\" # remove-before-flight' >> /target/etc/default/grub"
+    - "curtin in-target --target=/target -- update-grub"
+
+    # Copy /cdrom/ssh-config to /target/etc/ssh, if it exists
+    - "! [ -d /cdrom/ssh-config ] || (mkdir -p /target/etc/ssh && cp -r /cdrom/ssh-config/* /target/etc/ssh)"
+
+  shutdown: reboot
+
+  user-data:
+    bootcmd:
+      - "bash /sp-bootstrap/hook.sh early-welcome"
+
+    write_files:
+      - content: |
+          [daemon]
+          AutomaticLoginEnable=True
+          AutomaticLogin=ubuntu
+        path: /etc/gdm3/custom.conf
+        append: true
+      - content: |
+          [org.gnome.settings-daemon.plugins.power]
+          sleep-inactive-ac-timeout=0
+          sleep-inactive-battery-timeout=0
+          sleep-inactive-battery-type='nothing'
+          sleep-inactive-ac-type='nothing'
+          idle-dim=false
+          [org.gnome.desktop.session]
+          idle-delay=0
+          [org.gnome.desktop.screensaver]
+          ubuntu-lock-on-suspend=false
+          lock-enabled=false
+          idle-activation-enabled=false
+        path: /usr/share/glib-2.0/schemas/certification.gschema.override
+
+    users:
+      - name: ubuntu
+        sudo: "ALL=(ALL) NOPASSWD:ALL"
+        lock_passwd: false
+        shell: /bin/bash
+        # this is just "ubuntu"
+        passwd: "$6$rounds=4096$PCrfo.ggdf4ubP$REjyaoY2tUWH2vjFJjvLs3rDxVTszGR9P7mhH9sHb2MsELfc53uV/v15jDDOJU/9WInfjjTKJPlD5URhX5Mix0"
+
+    locale: en_US.UTF-8
+
+    packages:
+      - openssh-server
+
+    package_upgrade: false
+
+    runcmd:
+      - ["glib-compile-schemas", "/usr/share/glib-2.0/schemas"]
+      - ["sudo", "-u", "ubuntu", "-H", "gsettings", "reset-recursively", "org.gnome.settings-daemon.plugins.power"]
+      - ["sudo", "-u", "ubuntu", "-H", "gsettings", "reset-recursively", "org.gnome.desktop.session"]
+      - ["sudo", "-u", "ubuntu", "-H", "gsettings", "reset-recursively", "org.gnome.desktop.screensaver"]
+
+    # Reboot after early-welcome is done
+    power_state:
+      mode: "reboot"
+      message: "early-welcome setup complete, rebooting..."
+      timeout: 30
+
+bootcmd:
+  - ['plymouth', 'display-message', '--text', 'Starting automated testing installer...']

device-connectors/src/testflinger_device_connectors/data/muxpi/oem_autoinstall/provision-image.sh → device-connectors/src/testflinger_device_connectors/data/oem_autoinstall/provision-image.sh

@@ -22,7 +22,7 @@ EOF
 
 if [ $# -lt 3 ]; then
     usage
-    exit
+    exit 1
 fi
 
 TARGET_USER="ubuntu"
@@ -212,7 +212,7 @@ do
 
     if [ -z "$STORE_PART" ]; then
         echo "Can't find partition to store ISO on target $addr"
-        exit
+        exit 1
     fi
     RESET_PART="${STORE_PART:0:-1}2"
     RESET_PARTUUID=$($SSH "$TARGET_USER"@"$addr" -- lsblk -n -o PARTUUID "$RESET_PART")
@@ -299,29 +299,6 @@ do
     fi
 done
 
-# Polling the targets
-STARTED=("${TARGET_IPS[@]}")
-finished=0
-startTime=$(date +%s)
-while :;
-do
-    sleep 180
-    currentTime=$(date +%s)
-    if [[ $((currentTime - startTime)) -gt $TIMEOUT ]]; then
-        echo "Timeout is reached, deployment was not finished"
-        break
-    fi
-
-    for addr in "${STARTED[@]}";
-    do
-        if $SSH "$TARGET_USER"@"$addr" -- exit; then
-            STARTED=("${STARTED[@]/$addr}")
-            finished=$((finished + 1))
-        fi
-    done
-
-    if [ $finished -eq ${#TARGET_IPS[@]} ]; then
-        echo "Deployment is done"
-        break
-    fi
-done
+echo "Deployment will start after reboot"
+exit 0
+# Let device connector to poll the status

device-connectors/src/testflinger_device_connectors/devices/oem_autoinstall/oem_autoinstall.py

@@ -25,6 +25,7 @@
 import subprocess
 import yaml
 import shutil
+import time
 
 from testflinger_device_connectors.devices import (
     ProvisioningError,
@@ -44,16 +45,17 @@ def __init__(self, config, job_data):
             self.config = yaml.safe_load(configfile)
         with open(job_data, encoding="utf-8") as job_json:
             self.job_data = json.load(job_json)
+        self.data_path = Path(__file__).parent / "../../data/oem_autoinstall"
 
     def provision(self):
         """Provision the device"""
 
         # Ensure the device is online and reachable
         try:
-            self.test_ssh_access()
+            self.copy_ssh_id()
         except subprocess.CalledProcessError:
             self.hardreset()
-            self.test_ssh_access()
+            self.check_device_booted()
 
         provision_data = self.job_data.get("provision_data", {})
         image_url = provision_data.get("url")
@@ -69,10 +71,13 @@ def provision(self):
             raise ProvisioningError("No image url provided")
 
         if not user_data:
-            logger.error(
-                "Please provide user-data file in provision_data section"
-            )
-            raise ProvisioningError("No user-data provided")
+            logger.info("No user-data provided, using default user-data file")
+            default_user_data = self.data_path / "default-user-data"
+
+            if not default_user_data.exists():
+                raise ProvisioningError("Default user-data file not found")
+            shutil.copy(default_user_data, ATTACHMENTS_PROV_DIR)
+            user_data = "default-user-data"
 
         # provision-image.sh expects specific filename,
         # so need to rename if doesn't match
@@ -89,6 +94,7 @@ def provision(self):
             token_file_path = "url_token"
             self.copy_to_deploy_path(token_file, token_file_path)
         self.run_deploy_script(image_url)
+        self.check_device_booted()
 
     def copy_to_deploy_path(self, source_path, dest_path):
         """
@@ -116,15 +122,19 @@ def copy_to_deploy_path(self, source_path, dest_path):
     def run_deploy_script(self, image_url):
         """Run the script to deploy ISO and config files"""
         device_ip = self.config["device_ip"]
+        test_username = self.get_test_data_or_default(
+            "test_username", "ubuntu"
+        )
 
-        data_path = Path(__file__).parent / "../../data/muxpi/oem_autoinstall"
         logger.info("Running deployment script")
 
-        deploy_script = data_path / "provision-image.sh"
+        deploy_script = self.data_path / "provision-image.sh"
         cmd = [
             deploy_script,
             "--iso-dut",
             image_url,
+            "-u",
+            test_username,
             "--local-config",
             ATTACHMENTS_PROV_DIR,
             device_ip,
@@ -168,6 +178,38 @@ def test_ssh_access(self):
             logger.error("SSH connection failed: %s", result.stderr)
             raise ProvisioningError("Failed SSH to DUT")
 
+    def get_test_data_or_default(self, attribute, default_value):
+        """Helper function to safely get test attributes"""
+        try:
+            return self.job_data.get("test_data", {}).get(
+                attribute, default_value
+            )
+        except AttributeError:
+            return default_value
+
+    def copy_ssh_id(self):
+        """Copy the ssh id to the device"""
+
+        test_username = self.get_test_data_or_default(
+            "test_username", "ubuntu"
+        )
+        test_password = self.get_test_data_or_default(
+            "test_password", "ubuntu"
+        )
+
+        cmd = [
+            "sshpass",
+            "-p",
+            test_password,
+            "ssh-copy-id",
+            "-o",
+            "StrictHostKeyChecking=no",
+            "-o",
+            "UserKnownHostsFile=/dev/null",
+            f"{test_username}@{self.config['device_ip']}",
+        ]
+        subprocess.check_output(cmd, stderr=subprocess.STDOUT, timeout=60)
+
     def hardreset(self):
         """
         Reboot the device.
@@ -185,3 +227,22 @@ def hardreset(self):
                 subprocess.check_call(cmd.split(), timeout=120)
             except subprocess.SubprocessError as exc:
                 raise RecoveryError("Error running reboot script!") from exc
+
+    def check_device_booted(self):
+        """Check to see if the device is booted and reachable with ssh"""
+        logger.info("Checking to see if the device is available.")
+        started = time.time()
+        # Wait for provisioning to complete - can take a very long time
+        while time.time() - started < 5400:
+            try:
+                time.sleep(90)
+                self.copy_ssh_id()
+                return True
+            except subprocess.SubprocessError:
+                pass
+        # If we get here, then we didn't boot in time
+        agent_name = self.config.get("agent_name")
+        logger.error(
+            "Device %s unreachable,  provisioning" "failed!", agent_name
+        )
+        raise ProvisioningError("Failed to boot test image!")

docs/reference/device-connector-types.rst

@@ -304,9 +304,10 @@ The ``oem_autoinstall`` device connector supports the following ``provision_data
           pass = $PASSWORD
 
     * - ``user_data``
-      - Required file provided with :ref:`file attachments <file_attachments>`.
+      - Optional file provided with :ref:`file attachments <file_attachments>`.
         This file will be consumed by the autoinstall and cloud-init.
-        Sample user-data is provided in the section below.
+        Sample user-data is provided in the section below. When file is missing
+        connector will use the default-user-data file.
     * - ``redeploy_cfg``
       - Optional file provided with :ref:`file attachments <file_attachments>`.
         This file will override the grub.cfg in reset partition.