From 3468b22374d42fd4239bfe8a6687a339bc58b3c3 Mon Sep 17 00:00:00 2001 From: jEzEk Date: Sun, 12 Sep 2021 17:33:39 +0200 Subject: [PATCH 01/10] Changes needed to use crossbuilder on manjaro - If no 'dpkg' command found, try to determine host architecture using other metod(s). - Don't check for 'dpkg-architecture' command at beginning. Run the command in container, if missing. - Don't check for 'dpkg-parsechangelog' command at beginning. Run the command in container or replace it by using grep+sed, if missing. - Run lxd daemon at startup, if not running. - Use current user/group id in lxd subuid/subgid setup & assurance. - If not 'apt' found, check for 'pacman' and use if present. - If container is missing network access, try to apply 'dhclient' hack. and then check again for network. --- crossbuilder | 269 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 233 insertions(+), 36 deletions(-) diff --git a/crossbuilder b/crossbuilder index a5b710c..6be8e0f 100755 --- a/crossbuilder +++ b/crossbuilder @@ -145,15 +145,25 @@ variables () { SOURCE_PATH_LOCAL=$MOUNTED_DIRECTORY SOURCE_PATH_CONTAINER=$MOUNT_POINT POST_DEPLOY_SCRIPT=.crossbuilder/post_deploy - HOST_FARCH=$(dpkg-architecture -f -a$HOST_ARCH -qDEB_BUILD_MULTIARCH) - TARGET_FARCH=$(dpkg-architecture -f -a$TARGET_ARCH -qDEB_HOST_MULTIARCH) - if [ "$(dpkg-architecture -f -a$TARGET_ARCH -qDEB_BUILD_ARCH)" != "$TARGET_ARCH" ]; then - DEB_BUILD_PROFILES="cross nocheck nodoc noudeb" - EXTRA_DEB_BUILD_OPTIONS="nocheck nodoc noudeb" - else - DEB_BUILD_PROFILES="noudeb" - EXTRA_DEB_BUILD_OPTIONS="noudeb" + + # To fill the HOST_FARCH, TARGET_FARCH, DEB_BUILD_PROFILES and EXTRA_DEB_BUILD_OPTIONS variables, + # you need the `dpkg-architecture' command, which is not always installed on host macine, + # but is present in the ubports sdk container. + # So if there is no 'dpkg-architecture' command, don't worry, the variables will be filed later, upon container is ensured. + if which dpkg-architecture > /dev/null 2>&1 ; then + HOST_FARCH=$(dpkg-architecture -f -a$HOST_ARCH -qDEB_BUILD_MULTIARCH) + TARGET_FARCH=$(dpkg-architecture -f -a$TARGET_ARCH -qDEB_HOST_MULTIARCH) + if [ "$(dpkg-architecture -f -a$TARGET_ARCH -qDEB_BUILD_ARCH)" != "$TARGET_ARCH" ]; then + # If you're changing these variables, please change them in the 'detect_farch_and_build_params_in_container_if_needed' function too. + DEB_BUILD_PROFILES="cross nocheck nodoc noudeb" + EXTRA_DEB_BUILD_OPTIONS="nocheck nodoc noudeb" + else + # If you're changing these variables, please change them in the 'detect_farch_and_build_params_in_container_if_needed' function too. + DEB_BUILD_PROFILES="noudeb" + EXTRA_DEB_BUILD_OPTIONS="noudeb" + fi fi + TMP_DIR="$(mktemp -d --tmpdir crossbuilder.XXXXXX)" trap "rm -r $TMP_DIR" HUP INT TERM QUIT EXIT } @@ -167,6 +177,12 @@ check_lxd_accessible () { exit 1 fi + # On some distros the lxd is not runnig by default. Start if not started. + if which systemctl > /dev/null 2>&1 && ! sudo systemctl status lxd > /dev/null 2>&1 ; then + echo "sudo systemctl start lxd" + sudo systemctl start lxd + fi + if ! lxc info > /dev/null 2>&1 ; then echo -e "${ERROR_COLOR}LXD was installed but is not accessible." echo "Check the 'lxd' group exists and you are a member, then restart your computer." @@ -186,30 +202,52 @@ lxd_has_image_or_container () { } is_subuid_setup () { - if ! grep "root:1000:1" /etc/subuid > /dev/null ; then + local USER_ID=$1 + local GROUP_ID=$2 + if ! grep "root:${USER_ID}:1" /etc/subuid > /dev/null ; then return 1 fi - if ! grep "root:1000:1" /etc/subgid > /dev/null ; then + if ! grep "root:${GROUP_ID}:1" /etc/subgid > /dev/null ; then return 1 fi return 0 } ensure_lxd_subuid () { - if test -z "$FORCE_PRIVILEGED" && ! is_subuid_setup ; then + # Fetch user id and his group id to correctly setup subuid/subgid. + local USER_ID=$(id | sed 's/.*uid=\([0-9]\+\).*/\1/') + local GROUP_ID=$(id | sed 's/.*gid=\([0-9]\+\).*/\1/') + if test -z "$FORCE_PRIVILEGED" && ! is_subuid_setup $USER_ID $GROUP_ID ; then echo -e "${ERROR_COLOR}LXD requires subuid to be setup adequately to mount directories in containers.${NC}" echo -e "${POSITIVE_COLOR}Would you like to do that now? (y/n) ${NC}" read REPLY echo if [ "$REPLY" = y ] then - sudo usermod --add-subuids 1000-1000 root - sudo usermod --add-subgids 1000-1000 root - if which service > /dev/null ; then + # Setup subuid. + if [ ! -f "/etc/subuid" ] ; then + sudo touch /etc/subuid + fi + sudo usermod --add-subuids "10000-75536" `whoami` + sudo usermod --add-subuids "${USER_ID}-${USER_ID}" root + sudo usermod --add-subuids "10000-75536" root + + # Setup subgid. + if [ ! -f "/etc/subgid" ] ; then + sudo touch /etc/subgid + fi + sudo usermod --add-subgids "10000-75536" `whoami` + sudo usermod --add-subgids "${GROUP_ID}-${GROUP_ID}" root + sudo usermod --add-subgids "10000-75536" root + + if which service > /dev/null 2>&1; then + echo "sudo service lxd restart" sudo service lxd restart - elif which snap > /dev/null && snap connections lxd > /dev/null ; then + elif which snap > /dev/null 2>&1 && snap connections lxd > /dev/null 2>&1 ; then + echo "sudo snap restart lxd" sudo snap restart lxd - else + elif which systemctl > /dev/null 2>&1 ; then + echo "sudo systemctl restart lxd" sudo systemctl restart lxd fi else @@ -250,7 +288,16 @@ setup_lxd () { then if ! which zpool > /dev/null ; then echo -e "${POSITIVE_COLOR}Installing ZFS.${NC}" - sudo apt-get install -y zfsutils-linux + if which apt-get > /dev/null 2>&1 ; then + echo "sudo apt-get install -y zfsutils-linux" + sudo apt-get install -y zfsutils-linux + elif which pacman > /dev/null 2>&1 ; then + echo "sudo pacman -Syu zfs-utils" + sudo pacman -Syu zfs-utils + else + echo -e "${ERROR_COLOR}No known package manager found to install zfs utilities.${NC}" + exit 1 + fi fi LXD_POOL=~/zfs/lxd.img echo -e "${POSITIVE_COLOR}Creating file $LXD_POOL to contain all of LXD's images and containers.${NC}" @@ -350,6 +397,22 @@ nonsdk_container_setup () { esac } +detect_farch_and_build_params_in_container_if_needed () { + if [ -z $TARGET_FARCH ] || [ -z $HOST_FARCH ] ; then + HOST_FARCH=$(exec_container dpkg-architecture -f -a$HOST_ARCH -qDEB_BUILD_MULTIARCH) + TARGET_FARCH=$(exec_container dpkg-architecture -f -a$TARGET_ARCH -qDEB_HOST_MULTIARCH) + if [ "$(exec_container dpkg-architecture -f -a$TARGET_ARCH -qDEB_BUILD_ARCH)" != "$TARGET_ARCH" ]; then + # If you're changing these variables, please change them in the 'variables' function too. + DEB_BUILD_PROFILES="cross nocheck nodoc noudeb" + EXTRA_DEB_BUILD_OPTIONS="nocheck nodoc noudeb" + else + # If you're changing these variables, please change them in the 'variables' function too. + DEB_BUILD_PROFILES="noudeb" + EXTRA_DEB_BUILD_OPTIONS="noudeb" + fi + fi +} + create_container () { lxc remote --protocol=simplestreams --public=true --accept-certificate=true add ubports-sdk https://sdk-images.ubports.com || true lxc init $LXD_IMAGE $LXD_CONTAINER $EPHEMERAL_FLAG @@ -386,6 +449,8 @@ create_container () { exec_container_root adduser $USERNAME sudo # set empty password for the user exec_container_root passwd --delete $USERNAME + # There is a chance, that some farch and build variables were not filled, due to missing packages on host machine. Use container to fill them if needed. + detect_farch_and_build_params_in_container_if_needed exec_container_root "printf 'export PKG_CONFIG_PATH=/usr/lib/$TARGET_FARCH/pkgconfig\n\ export CROSS_COMPILE=$TARGET_FARCH-\n\ export CC=$TARGET_FARCH-gcc\n\ @@ -401,6 +466,9 @@ ensure_container () { echo -e "${POSITIVE_COLOR}LXD container $LXD_CONTAINER already exists.${NC}" config_container_dir_mount start_container + check_for_container_network + # There is a chance, that some farch and build variables were not filled, due to missing packages on host machine. Use container to fill them if needed. + detect_farch_and_build_params_in_container_if_needed else echo -e "${POSITIVE_COLOR}Creating LXD container $LXD_CONTAINER using $LXD_IMAGE.${NC}" create_container @@ -458,7 +526,8 @@ get_source_package () { exit 1 fi - PACKAGE_VERSION=`dpkg-parsechangelog --show-field Version -l $PACKAGE*/debian/changelog` + PACKAGE_VERSION=$(exec_container dpkg-parsechangelog --show-field Version -l $PACKAGE*/debian/changelog) + exec_container "mv $PACKAGE*.orig.tar.* $PACKAGE*diff.* $PACKAGE*debian.tar* $PACKAGE*.dsc $USERDIR/" || true mv $PACKAGE-*/* . mv $PACKAGE-*/.[!.]* . || true @@ -550,6 +619,12 @@ install_dependencies () { cp debian/control "${TMP_DIR}/" workaround_multi_arch_deps ${TMP_DIR}/control exec_container_root apt-get update + + # There is a chance, that some farch and build variables were not filled, due to missing packages on host machine. Use container to fill them if needed. + # The 'install_dependencies' function is always called after 'ensure_container', which calls the 'detect_farch_and_build_params_in_container_if_needed' function. + # So it is not needed here. But just to be sure, let's call it anyway. + detect_farch_and_build_params_in_container_if_needed + # TODO: add build profiles to mk-build-deps once wishlist bug # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=894732 is fixed and # propagated to Ubuntu version we will use in the future. @@ -631,6 +706,11 @@ build_make_install () { } build () { + # There is a chance, that some farch and build variables were not filled, due to missing packages on host machine. Use container to fill them if needed. + # The 'build' function is always called after 'ensure_container', which calls the 'detect_farch_and_build_params_in_container_if_needed' function. + # So it is not needed here. But just to be sure, let's call it anyway. + detect_farch_and_build_params_in_container_if_needed + if [ -n "$NO_DEB" ] ; then build_make_install else @@ -671,8 +751,22 @@ check_for_container_network() { sleep 1 done if [ $NETWORK_UP -ne 1 ] ; then - echo -e "${ERROR_COLOR}Container is not connected to the Internet.${NC}" - exit 1 + # On manjaro the 16.04 containers can't get ip4 address. + # This can be corrected by running 'dhclient' in container. + exec_container_root "dhclient" + for i in `seq 1 10` + do + if lxc query "/1.0/containers/${LXD_CONTAINER}/state" | jq -e '.network.eth0.addresses | any( .family == "inet" )' > /dev/null 2>&1 ; then + echo -e "${POSITIVE_COLOR}Container is connected to the Internet.${NC}" + NETWORK_UP=1 + break + fi + sleep 1 + done + if [ $NETWORK_UP -ne 1 ] ; then + echo -e "${ERROR_COLOR}Container is not connected to the Internet.${NC}" + exit 1 + fi fi } @@ -797,14 +891,12 @@ deploy_to_device () { } MISSING_PACKAGES= -if ! which dpkg > /dev/null ; then - MISSING_PACKAGES="dpkg" -fi -if ! which dpkg-parsechangelog > /dev/null ; then - MISSING_PACKAGES="dpkg-dev $MISSING_PACKAGES" -fi -if ! which adb > /dev/null ; then - MISSING_PACKAGES="android-tools-adb $MISSING_PACKAGES" +if ! which adb > /dev/null 2>&1 ; then + if which apt > /dev/null 2>&1 ; then + MISSING_PACKAGES="android-tools-adb $MISSING_PACKAGES" + elif which pacman > /dev/null 2>&1 ; then + MISSING_PACKAGES="android-tools $MISSING_PACKAGES" + fi fi if ! which jq > /dev/null; then MISSING_PACKAGES="jq $MISSING_PACKAGES" @@ -812,8 +904,16 @@ fi if [ ! -z "$MISSING_PACKAGES" ] ; then echo -e "${POSITIVE_COLOR}$PROGRAM_NAME depends on $MISSING_PACKAGES. Installing:${NC}" - echo "sudo apt install $MISSING_PACKAGES" - sudo apt install $MISSING_PACKAGES + if which apt > /dev/null 2>&1 ; then + echo "sudo apt install $MISSING_PACKAGES" + sudo apt install $MISSING_PACKAGES + elif which pacman > /dev/null 2>&1 ; then + echo "sudo pacman -Syu $MISSING_PACKAGES" + sudo pacman -Syu $MISSING_PACKAGES + else + echo -e "${ERROR_COLOR}No known package manager found to install missing packages.${NC}" + exit 1 + fi fi if stat --file-system $HOME | grep ecrypt ; then @@ -831,8 +931,21 @@ if ! which lxd > /dev/null ; then echo if [ "$REPLY" = y ] then - echo "sudo apt-get install -y lxd lxd-client" - sudo apt-get install -y lxd lxd-client + if which apt-get > /dev/null 2>&1 ; then + echo "sudo apt-get install -y lxd lxd-client" + sudo apt-get install -y lxd lxd-client + elif which pacman > /dev/null 2>&1 ; then + echo "sudo pacman -Syu --noconfirm lxd" + sudo pacman -Syu lxd + echo "sudo systemctl start lxd" + sudo systemctl start lxd + elif which snap > /dev/null 2>&1 ; then + echo "sudo snap install lxd" + sudo snap install lxd + else + echo -e "${ERROR_COLOR}No known package manager found to install lxd containers.${NC}" + exit 1 + fi setup_lxd ensure_lxd_subuid newgrp lxd @@ -845,10 +958,79 @@ if ! which lxd > /dev/null ; then fi fi -check_lxd_accessible ensure_lxd_subuid +check_lxd_accessible + +if [ -z $HOST_ARCH ] ; then + if which dpkg > /dev/null 2>&1 ; then + HOST_ARCH=`dpkg --print-architecture` + else + # To find out the host's debian architecture name, first find out debian multiarch tuple and use conversion table. + # Source: https://wiki.debian.org/Multiarch/Tuples + + if [ -z $HOST_FARCH ] && which gcc > /dev/null 2>&1 ; then + HOST_FARCH=$(gcc -print-multiarch) + + # On manjaro the 'gcc -print-multiarch' returns nothing. + # Use the 'gcc -dumpmachine' multiarch triplet result and strip the vendor part if the result contains it. + [ -z $HOST_FARCH ] && HOST_FARCH=$(gcc -dumpmachine | sed 's/^\([^-]\+\)-\([^-]\+\)-\([^-]\+\)-\([^-]\+\)$/\1-\3-\4/') + fi + #echo "HOST_FARCH=\"${HOST_FARCH}\"" + + declare -A MULTIARCH_TO_DEBIAN=( \ + [alpha-linux-gnu]=alpha \ + [x86_64-linux-gnu]=amd64 \ + [arc-linux-gnu]=arc \ + [arm-linux-gnu]=arm \ + [aarch64-linux-gnu]=arm64 \ + [aarch64-linux-gnu_ilp32]=arm64ilp32 \ + [arm-linux-gnueabi]=armel \ + [arm-linux-gnueabihf]=armhf \ + [hppa-linux-gnu]=hppa \ + [i386-gnu]=hurd-i386 \ + [i386-linux-gnu]=i386 \ + [ia64-linux-gnu]=ia64 \ + [x86_64-kfreebsd-gnu]=kfreebsd-amd64 \ + [i386-kfreebsd-gnu]=kfreebsd-i386 \ + [m68k-linux-gnu]=m68k \ + [mips-linux-gnu]=mips \ + [mipsel-linux-gnu]=mipsel \ + [mips64-linux-gnuabi64]=mips64 \ + [mips64el-linux-gnuabi64]=mips64el \ + [mips64-linux-gnuabin32]=mipsn32 \ + [mips64el-linux-gnuabin32]=mipsn32el \ + [mipsisa32r6-linux-gnu]=mipsr6 \ + [mipsisa32r6el-linux-gnu]=mipsr6el \ + [mipsisa64r6-linux-gnuabi64]=mips64r6 \ + [mipsisa64r6el-linux-gnuabi64]=mips64r6el \ + [mipsisa64r6-linux-gnuabin32]=mipsn32r6 \ + [mipsisa64r6el-linux-gnuabin32]=mipsn32r6el \ + [powerpc-linux-gnu]=powerpc \ + [powerpc-linux-gnuspe]=powerpcspe \ + [powerpc64-linux-gnu]=ppc64 \ + [powerpc64le-linux-gnu]=ppc64el \ + [riscv64-linux-gnu]=riscv64 \ + [s390-linux-gnu]=s390 \ + [s390x-linux-gnu]=s390x \ + [sh4-linux-gnu]=sh4 \ + [sparc-linux-gnu]=sparc \ + [sparc64-linux-gnu]=sparc64 \ + [x86_64-uefi]=uefi-amd6436 \ + [aarch64-uefi]=uefi-arm6436 \ + [arm-uefi]=uefi-armhf36 \ + [i386-uefi]=uefi-i38636 \ + [x86_64-linux-gnux32]=x32 \ + ) + HOST_ARCH=${MULTIARCH_TO_DEBIAN[$HOST_FARCH]} + fi +fi + +if [ -z $HOST_ARCH ] ; then + echo -e "${ERROR_COLOR}Coudln't find out debian architecture name of this device.${NC}" + echo -e "If you know it, set the HOST_ARCH variable before running crossbuilder." + exit 1 +fi -HOST_ARCH=`dpkg --print-architecture` TARGET_ARCH=armhf TARGET_UBUNTU=16.04 @@ -958,8 +1140,21 @@ check_command_parameter_count () { detect_package () { if test -e debian/changelog ; then - PACKAGE=`dpkg-parsechangelog --show-field Source` - PACKAGE_VERSION=`dpkg-parsechangelog --show-field Version` + if which dpkg-parsechangelog > /dev/null 2>&1 ; then + PACKAGE=`dpkg-parsechangelog --show-field Source` + PACKAGE_VERSION=`dpkg-parsechangelog --show-field Version` + else + # Get package name and version parsing the debian/changelog file using grep. + local RES=($(grep -P '^[^ ]+ \([^)]+\) [^;]+; urgency=.*$' debian/changelog | head -n1 | sed 's/^\([^ ]\+\) (\([^)]\+\)).*/\1 \2/')) + PACKAGE=${RES[0]} + PACKAGE_VERSION=${RES[1]} + #echo "PACKAGE=\"${PACKAGE}\"" + #echo "PACKAGE_VERSION=\"${PACKAGE_VERSION}\"" + if [ -z $PACKAGE ] || [ -z $PACKAGE_VERSION ] ; then + echo -e "${ERROR_COLOR}Coudln't find out package name and/or version from debian/changelog file.${NC}" + exit 1 + fi + fi fi } @@ -1137,6 +1332,8 @@ else enter_or_detect_package $1 variables start_container_if_exists + # There is a chance, that some farch and build variables were not filled, due to missing packages on host machine. Use container to fill them if needed. + detect_farch_and_build_params_in_container_if_needed clean ;; deploy) From 0441ac11d1d0c100b26a83e22cb9a83d001fc94a Mon Sep 17 00:00:00 2001 From: jEzEk Date: Wed, 29 Sep 2021 18:14:16 +0200 Subject: [PATCH 02/10] Container ensurement comment clarification. --- crossbuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crossbuilder b/crossbuilder index 6be8e0f..812f844 100755 --- a/crossbuilder +++ b/crossbuilder @@ -149,7 +149,7 @@ variables () { # To fill the HOST_FARCH, TARGET_FARCH, DEB_BUILD_PROFILES and EXTRA_DEB_BUILD_OPTIONS variables, # you need the `dpkg-architecture' command, which is not always installed on host macine, # but is present in the ubports sdk container. - # So if there is no 'dpkg-architecture' command, don't worry, the variables will be filed later, upon container is ensured. + # So if there is no 'dpkg-architecture' command, don't worry, the variables will be filed later, upon container is ensured (in the "ensure_container" function). if which dpkg-architecture > /dev/null 2>&1 ; then HOST_FARCH=$(dpkg-architecture -f -a$HOST_ARCH -qDEB_BUILD_MULTIARCH) TARGET_FARCH=$(dpkg-architecture -f -a$TARGET_ARCH -qDEB_HOST_MULTIARCH) From f1d37c77d900d5c74e575a5bc0c4995dfb9fa840 Mon Sep 17 00:00:00 2001 From: jEzEk Date: Wed, 29 Sep 2021 18:36:03 +0200 Subject: [PATCH 03/10] Move "dpkg-architecture" command usage entirely into container --- crossbuilder | 66 ++++++++++++++-------------------------------------- 1 file changed, 17 insertions(+), 49 deletions(-) diff --git a/crossbuilder b/crossbuilder index 812f844..b3710a6 100755 --- a/crossbuilder +++ b/crossbuilder @@ -145,27 +145,12 @@ variables () { SOURCE_PATH_LOCAL=$MOUNTED_DIRECTORY SOURCE_PATH_CONTAINER=$MOUNT_POINT POST_DEPLOY_SCRIPT=.crossbuilder/post_deploy - - # To fill the HOST_FARCH, TARGET_FARCH, DEB_BUILD_PROFILES and EXTRA_DEB_BUILD_OPTIONS variables, - # you need the `dpkg-architecture' command, which is not always installed on host macine, - # but is present in the ubports sdk container. - # So if there is no 'dpkg-architecture' command, don't worry, the variables will be filed later, upon container is ensured (in the "ensure_container" function). - if which dpkg-architecture > /dev/null 2>&1 ; then - HOST_FARCH=$(dpkg-architecture -f -a$HOST_ARCH -qDEB_BUILD_MULTIARCH) - TARGET_FARCH=$(dpkg-architecture -f -a$TARGET_ARCH -qDEB_HOST_MULTIARCH) - if [ "$(dpkg-architecture -f -a$TARGET_ARCH -qDEB_BUILD_ARCH)" != "$TARGET_ARCH" ]; then - # If you're changing these variables, please change them in the 'detect_farch_and_build_params_in_container_if_needed' function too. - DEB_BUILD_PROFILES="cross nocheck nodoc noudeb" - EXTRA_DEB_BUILD_OPTIONS="nocheck nodoc noudeb" - else - # If you're changing these variables, please change them in the 'detect_farch_and_build_params_in_container_if_needed' function too. - DEB_BUILD_PROFILES="noudeb" - EXTRA_DEB_BUILD_OPTIONS="noudeb" - fi - fi - TMP_DIR="$(mktemp -d --tmpdir crossbuilder.XXXXXX)" trap "rm -r $TMP_DIR" HUP INT TERM QUIT EXIT + + unset HOST_FARCH TARGET_FARCH DEB_BUILD_PROFILES EXTRA_DEB_BUILD_OPTIONS + # To set the HOST_FARCH, TARGET_FARCH, DEB_BUILD_PROFILES and EXTRA_DEB_BUILD_OPTIONS variables, + # you need to call the "ensure_container" function. } check_lxd_accessible () { @@ -397,20 +382,16 @@ nonsdk_container_setup () { esac } -detect_farch_and_build_params_in_container_if_needed () { - if [ -z $TARGET_FARCH ] || [ -z $HOST_FARCH ] ; then - HOST_FARCH=$(exec_container dpkg-architecture -f -a$HOST_ARCH -qDEB_BUILD_MULTIARCH) - TARGET_FARCH=$(exec_container dpkg-architecture -f -a$TARGET_ARCH -qDEB_HOST_MULTIARCH) - if [ "$(exec_container dpkg-architecture -f -a$TARGET_ARCH -qDEB_BUILD_ARCH)" != "$TARGET_ARCH" ]; then - # If you're changing these variables, please change them in the 'variables' function too. - DEB_BUILD_PROFILES="cross nocheck nodoc noudeb" - EXTRA_DEB_BUILD_OPTIONS="nocheck nodoc noudeb" - else - # If you're changing these variables, please change them in the 'variables' function too. - DEB_BUILD_PROFILES="noudeb" - EXTRA_DEB_BUILD_OPTIONS="noudeb" - fi - fi +detect_farch_and_build_params_in_container () { + HOST_FARCH=$(exec_container dpkg-architecture -f -a$HOST_ARCH -qDEB_BUILD_MULTIARCH) + TARGET_FARCH=$(exec_container dpkg-architecture -f -a$TARGET_ARCH -qDEB_HOST_MULTIARCH) + if [ "$(exec_container dpkg-architecture -f -a$TARGET_ARCH -qDEB_BUILD_ARCH)" != "$TARGET_ARCH" ]; then + DEB_BUILD_PROFILES="cross nocheck nodoc noudeb" + EXTRA_DEB_BUILD_OPTIONS="nocheck nodoc noudeb" + else + DEB_BUILD_PROFILES="noudeb" + EXTRA_DEB_BUILD_OPTIONS="noudeb" + fi } create_container () { @@ -449,8 +430,7 @@ create_container () { exec_container_root adduser $USERNAME sudo # set empty password for the user exec_container_root passwd --delete $USERNAME - # There is a chance, that some farch and build variables were not filled, due to missing packages on host machine. Use container to fill them if needed. - detect_farch_and_build_params_in_container_if_needed + detect_farch_and_build_params_in_container exec_container_root "printf 'export PKG_CONFIG_PATH=/usr/lib/$TARGET_FARCH/pkgconfig\n\ export CROSS_COMPILE=$TARGET_FARCH-\n\ export CC=$TARGET_FARCH-gcc\n\ @@ -467,8 +447,7 @@ ensure_container () { config_container_dir_mount start_container check_for_container_network - # There is a chance, that some farch and build variables were not filled, due to missing packages on host machine. Use container to fill them if needed. - detect_farch_and_build_params_in_container_if_needed + detect_farch_and_build_params_in_container else echo -e "${POSITIVE_COLOR}Creating LXD container $LXD_CONTAINER using $LXD_IMAGE.${NC}" create_container @@ -620,11 +599,6 @@ install_dependencies () { workaround_multi_arch_deps ${TMP_DIR}/control exec_container_root apt-get update - # There is a chance, that some farch and build variables were not filled, due to missing packages on host machine. Use container to fill them if needed. - # The 'install_dependencies' function is always called after 'ensure_container', which calls the 'detect_farch_and_build_params_in_container_if_needed' function. - # So it is not needed here. But just to be sure, let's call it anyway. - detect_farch_and_build_params_in_container_if_needed - # TODO: add build profiles to mk-build-deps once wishlist bug # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=894732 is fixed and # propagated to Ubuntu version we will use in the future. @@ -706,11 +680,6 @@ build_make_install () { } build () { - # There is a chance, that some farch and build variables were not filled, due to missing packages on host machine. Use container to fill them if needed. - # The 'build' function is always called after 'ensure_container', which calls the 'detect_farch_and_build_params_in_container_if_needed' function. - # So it is not needed here. But just to be sure, let's call it anyway. - detect_farch_and_build_params_in_container_if_needed - if [ -n "$NO_DEB" ] ; then build_make_install else @@ -1332,8 +1301,7 @@ else enter_or_detect_package $1 variables start_container_if_exists - # There is a chance, that some farch and build variables were not filled, due to missing packages on host machine. Use container to fill them if needed. - detect_farch_and_build_params_in_container_if_needed + detect_farch_and_build_params_in_container clean ;; deploy) From a2f141fa808fa144cd7f5f5ae6e45da33944c9b8 Mon Sep 17 00:00:00 2001 From: jEzEk Date: Wed, 29 Sep 2021 18:39:32 +0200 Subject: [PATCH 04/10] Modify starting lxd echo text --- crossbuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crossbuilder b/crossbuilder index b3710a6..a649566 100755 --- a/crossbuilder +++ b/crossbuilder @@ -164,7 +164,7 @@ check_lxd_accessible () { # On some distros the lxd is not runnig by default. Start if not started. if which systemctl > /dev/null 2>&1 && ! sudo systemctl status lxd > /dev/null 2>&1 ; then - echo "sudo systemctl start lxd" + echo "Starting LXD" sudo systemctl start lxd fi From 22cb838767893e8310f4184511647ab43476a14f Mon Sep 17 00:00:00 2001 From: jEzEk Date: Wed, 29 Sep 2021 18:41:05 +0200 Subject: [PATCH 05/10] Simplify user and group id detection --- crossbuilder | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crossbuilder b/crossbuilder index a649566..bde9fa7 100755 --- a/crossbuilder +++ b/crossbuilder @@ -200,8 +200,8 @@ is_subuid_setup () { ensure_lxd_subuid () { # Fetch user id and his group id to correctly setup subuid/subgid. - local USER_ID=$(id | sed 's/.*uid=\([0-9]\+\).*/\1/') - local GROUP_ID=$(id | sed 's/.*gid=\([0-9]\+\).*/\1/') + local USER_ID=$(id -u) + local GROUP_ID=$(id -g) if test -z "$FORCE_PRIVILEGED" && ! is_subuid_setup $USER_ID $GROUP_ID ; then echo -e "${ERROR_COLOR}LXD requires subuid to be setup adequately to mount directories in containers.${NC}" echo -e "${POSITIVE_COLOR}Would you like to do that now? (y/n) ${NC}" From 11683b8bc1ccee1e0dd8b75fdd6cc824dc5a720a Mon Sep 17 00:00:00 2001 From: jEzEk Date: Fri, 1 Oct 2021 18:30:55 +0200 Subject: [PATCH 06/10] Detect package review fixes --- crossbuilder | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/crossbuilder b/crossbuilder index bde9fa7..3ef9f5f 100755 --- a/crossbuilder +++ b/crossbuilder @@ -1113,16 +1113,14 @@ detect_package () { PACKAGE=`dpkg-parsechangelog --show-field Source` PACKAGE_VERSION=`dpkg-parsechangelog --show-field Version` else - # Get package name and version parsing the debian/changelog file using grep. - local RES=($(grep -P '^[^ ]+ \([^)]+\) [^;]+; urgency=.*$' debian/changelog | head -n1 | sed 's/^\([^ ]\+\) (\([^)]\+\)).*/\1 \2/')) - PACKAGE=${RES[0]} - PACKAGE_VERSION=${RES[1]} - #echo "PACKAGE=\"${PACKAGE}\"" - #echo "PACKAGE_VERSION=\"${PACKAGE_VERSION}\"" - if [ -z $PACKAGE ] || [ -z $PACKAGE_VERSION ] ; then - echo -e "${ERROR_COLOR}Coudln't find out package name and/or version from debian/changelog file.${NC}" - exit 1 - fi + # Get package name and version parsing the debian/changelog file using grep. + local RES=($(grep -P '^[^ ]+ \([^)]+\) [^;]+; urgency=.*$' debian/changelog | head -n1 | sed 's/^\([^ ]\+\) (\([^)]\+\)).*/\1 \2/')) + PACKAGE=${RES[0]} + PACKAGE_VERSION=${RES[1]} + fi + if [ -z $PACKAGE ] || [ -z $PACKAGE_VERSION ] ; then + echo -e "${ERROR_COLOR}Coudln't find out package name and/or version from debian/changelog file.${NC}" + exit 1 fi fi } From 8e4e8a3fe7fc9dc71a7fb18ea46f1bfb0e1ff005 Mon Sep 17 00:00:00 2001 From: jEzEk Date: Tue, 5 Oct 2021 18:19:45 +0200 Subject: [PATCH 07/10] Move host architecture detection to function, remove exotic architectures --- crossbuilder | 98 ++++++++++++++++++---------------------------------- 1 file changed, 34 insertions(+), 64 deletions(-) diff --git a/crossbuilder b/crossbuilder index 3ef9f5f..803842a 100755 --- a/crossbuilder +++ b/crossbuilder @@ -859,6 +859,39 @@ deploy_to_device () { fi } +detect_host_architecture() { + if which dpkg > /dev/null 2>&1 ; then + HOST_ARCH=`dpkg --print-architecture` + else + # To find out the host's debian architecture name, first find out debian multiarch tuple and use conversion table. + # Source: https://wiki.debian.org/Multiarch/Tuples + + if [ -z $HOST_FARCH ] && which gcc > /dev/null 2>&1 ; then + HOST_FARCH=$(gcc -print-multiarch) + + # On manjaro the 'gcc -print-multiarch' returns nothing. + # Use the 'gcc -dumpmachine' multiarch triplet result and strip the vendor part if the result contains it. + [ -z $HOST_FARCH ] && HOST_FARCH=$(gcc -dumpmachine | sed 's/^\([^-]\+\)-\([^-]\+\)-\([^-]\+\)-\([^-]\+\)$/\1-\3-\4/') + fi + + declare -A MULTIARCH_TO_DEBIAN=( \ + [x86_64-linux-gnu]=amd64 \ + [arm-linux-gnu]=arm \ + [aarch64-linux-gnu]=arm64 \ + [aarch64-linux-gnu_ilp32]=arm64ilp32 \ + [arm-linux-gnueabi]=armel \ + [arm-linux-gnueabihf]=armhf \ + [riscv64-linux-gnu]=riscv64 \ + [x86_64-uefi]=uefi-amd6436 \ + [aarch64-uefi]=uefi-arm6436 \ + [arm-uefi]=uefi-armhf36 \ + [i386-uefi]=uefi-i38636 \ + [x86_64-linux-gnux32]=x32 \ + ) + HOST_ARCH=${MULTIARCH_TO_DEBIAN[$HOST_FARCH]} + fi +} + MISSING_PACKAGES= if ! which adb > /dev/null 2>&1 ; then if which apt > /dev/null 2>&1 ; then @@ -930,70 +963,7 @@ fi ensure_lxd_subuid check_lxd_accessible -if [ -z $HOST_ARCH ] ; then - if which dpkg > /dev/null 2>&1 ; then - HOST_ARCH=`dpkg --print-architecture` - else - # To find out the host's debian architecture name, first find out debian multiarch tuple and use conversion table. - # Source: https://wiki.debian.org/Multiarch/Tuples - - if [ -z $HOST_FARCH ] && which gcc > /dev/null 2>&1 ; then - HOST_FARCH=$(gcc -print-multiarch) - - # On manjaro the 'gcc -print-multiarch' returns nothing. - # Use the 'gcc -dumpmachine' multiarch triplet result and strip the vendor part if the result contains it. - [ -z $HOST_FARCH ] && HOST_FARCH=$(gcc -dumpmachine | sed 's/^\([^-]\+\)-\([^-]\+\)-\([^-]\+\)-\([^-]\+\)$/\1-\3-\4/') - fi - #echo "HOST_FARCH=\"${HOST_FARCH}\"" - - declare -A MULTIARCH_TO_DEBIAN=( \ - [alpha-linux-gnu]=alpha \ - [x86_64-linux-gnu]=amd64 \ - [arc-linux-gnu]=arc \ - [arm-linux-gnu]=arm \ - [aarch64-linux-gnu]=arm64 \ - [aarch64-linux-gnu_ilp32]=arm64ilp32 \ - [arm-linux-gnueabi]=armel \ - [arm-linux-gnueabihf]=armhf \ - [hppa-linux-gnu]=hppa \ - [i386-gnu]=hurd-i386 \ - [i386-linux-gnu]=i386 \ - [ia64-linux-gnu]=ia64 \ - [x86_64-kfreebsd-gnu]=kfreebsd-amd64 \ - [i386-kfreebsd-gnu]=kfreebsd-i386 \ - [m68k-linux-gnu]=m68k \ - [mips-linux-gnu]=mips \ - [mipsel-linux-gnu]=mipsel \ - [mips64-linux-gnuabi64]=mips64 \ - [mips64el-linux-gnuabi64]=mips64el \ - [mips64-linux-gnuabin32]=mipsn32 \ - [mips64el-linux-gnuabin32]=mipsn32el \ - [mipsisa32r6-linux-gnu]=mipsr6 \ - [mipsisa32r6el-linux-gnu]=mipsr6el \ - [mipsisa64r6-linux-gnuabi64]=mips64r6 \ - [mipsisa64r6el-linux-gnuabi64]=mips64r6el \ - [mipsisa64r6-linux-gnuabin32]=mipsn32r6 \ - [mipsisa64r6el-linux-gnuabin32]=mipsn32r6el \ - [powerpc-linux-gnu]=powerpc \ - [powerpc-linux-gnuspe]=powerpcspe \ - [powerpc64-linux-gnu]=ppc64 \ - [powerpc64le-linux-gnu]=ppc64el \ - [riscv64-linux-gnu]=riscv64 \ - [s390-linux-gnu]=s390 \ - [s390x-linux-gnu]=s390x \ - [sh4-linux-gnu]=sh4 \ - [sparc-linux-gnu]=sparc \ - [sparc64-linux-gnu]=sparc64 \ - [x86_64-uefi]=uefi-amd6436 \ - [aarch64-uefi]=uefi-arm6436 \ - [arm-uefi]=uefi-armhf36 \ - [i386-uefi]=uefi-i38636 \ - [x86_64-linux-gnux32]=x32 \ - ) - HOST_ARCH=${MULTIARCH_TO_DEBIAN[$HOST_FARCH]} - fi -fi - +[ -z $HOST_ARCH ] && detect_host_architecture if [ -z $HOST_ARCH ] ; then echo -e "${ERROR_COLOR}Coudln't find out debian architecture name of this device.${NC}" echo -e "If you know it, set the HOST_ARCH variable before running crossbuilder." From 449422a57abdeb2ec4c2c46b78bcf573ab2adafb Mon Sep 17 00:00:00 2001 From: jEzEk Date: Thu, 7 Oct 2021 18:29:30 +0200 Subject: [PATCH 08/10] Remove adding user sub[g,u]id at lxd initialization, fix root sub[u,g]id ranges --- crossbuilder | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crossbuilder b/crossbuilder index 803842a..9144c80 100755 --- a/crossbuilder +++ b/crossbuilder @@ -213,17 +213,15 @@ ensure_lxd_subuid () { if [ ! -f "/etc/subuid" ] ; then sudo touch /etc/subuid fi - sudo usermod --add-subuids "10000-75536" `whoami` sudo usermod --add-subuids "${USER_ID}-${USER_ID}" root - sudo usermod --add-subuids "10000-75536" root + sudo usermod --add-subuids "100000-165535" root # Setup subgid. if [ ! -f "/etc/subgid" ] ; then sudo touch /etc/subgid fi - sudo usermod --add-subgids "10000-75536" `whoami` sudo usermod --add-subgids "${GROUP_ID}-${GROUP_ID}" root - sudo usermod --add-subgids "10000-75536" root + sudo usermod --add-subgids "100000-165535" root if which service > /dev/null 2>&1; then echo "sudo service lxd restart" From dc04c2470f8d2d63471c2c0f5635d27dd504ec9a Mon Sep 17 00:00:00 2001 From: jEzEk Date: Sat, 16 Oct 2021 18:26:46 +0200 Subject: [PATCH 09/10] lxd service run detection rewrite, introducing lxd_service_action function --- crossbuilder | 59 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/crossbuilder b/crossbuilder index 9144c80..3e44d7e 100755 --- a/crossbuilder +++ b/crossbuilder @@ -153,6 +153,25 @@ variables () { # you need to call the "ensure_container" function. } +# Runs action, provided as first argument, on lxd service detected in system. +lxd_service_action () { + local action=$1 + + if which service > /dev/null 2>&1; then + echo "sudo service lxd ${action}" + sudo service lxd ${action} + elif which snap > /dev/null 2>&1 && snap connections lxd > /dev/null 2>&1 ; then + echo "sudo snap ${action} lxd" + sudo snap ${action} lxd + elif which systemctl > /dev/null 2>&1 && systemctl cat lxd > /dev/null 2>&1 ; then + echo "sudo systemctl ${action} lxd" + sudo systemctl ${action} lxd + else + echo -e "${ERROR_COLOR}Can't detect LXD service.${NC}" + return 1 + fi +} + check_lxd_accessible () { # /bin/lxd is replaced with a script which tells you to install the lxd snap # on later versions of Ubuntu. @@ -162,17 +181,21 @@ check_lxd_accessible () { exit 1 fi - # On some distros the lxd is not runnig by default. Start if not started. - if which systemctl > /dev/null 2>&1 && ! sudo systemctl status lxd > /dev/null 2>&1 ; then - echo "Starting LXD" - sudo systemctl start lxd - fi - - if ! lxc info > /dev/null 2>&1 ; then - echo -e "${ERROR_COLOR}LXD was installed but is not accessible." - echo "Check the 'lxd' group exists and you are a member, then restart your computer." - echo -e "For more information, run 'lxc info'.${NC}" - exit 1 + if ! LXC_INFO_OUTPUT=$(lxc info 2>&1) ; then + case "$LXC_INFO_OUTPUT" in + 'Error: Get "http://unix.socket/1.0":'*) + if ! lxd_service_action start || ! lxc info >/dev/null 2>&1 ; then + echo -e "${ERROR_COLOR}LXD service was not running and failed to start." + echo -e "For more information, run 'lxc info'.${NC}" + exit 1 + fi + ;; + *) + echo -e "${ERROR_COLOR}LXD was installed but is not accessible." + echo "Check the 'lxd' group exists and you are a member, then restart your computer." + echo -e "For more information, run 'lxc info'.${NC}" + exit 1 + esac fi } @@ -223,16 +246,7 @@ ensure_lxd_subuid () { sudo usermod --add-subgids "${GROUP_ID}-${GROUP_ID}" root sudo usermod --add-subgids "100000-165535" root - if which service > /dev/null 2>&1; then - echo "sudo service lxd restart" - sudo service lxd restart - elif which snap > /dev/null 2>&1 && snap connections lxd > /dev/null 2>&1 ; then - echo "sudo snap restart lxd" - sudo snap restart lxd - elif which systemctl > /dev/null 2>&1 ; then - echo "sudo systemctl restart lxd" - sudo systemctl restart lxd - fi + lxd_service_action restart else echo -e "${ERROR_COLOR}Unable to set up LXD for use in crossbuilder. Subuid setup refused.${NC}" exit 1 @@ -937,8 +951,7 @@ if ! which lxd > /dev/null ; then elif which pacman > /dev/null 2>&1 ; then echo "sudo pacman -Syu --noconfirm lxd" sudo pacman -Syu lxd - echo "sudo systemctl start lxd" - sudo systemctl start lxd + lxd_service_action start elif which snap > /dev/null 2>&1 ; then echo "sudo snap install lxd" sudo snap install lxd From 8d4d89c0b54a067e18dfe153058f1181e3da85e6 Mon Sep 17 00:00:00 2001 From: jEzEk Date: Wed, 20 Oct 2021 16:15:46 +0200 Subject: [PATCH 10/10] LXD service detection fixes --- crossbuilder | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crossbuilder b/crossbuilder index 3e44d7e..6799406 100755 --- a/crossbuilder +++ b/crossbuilder @@ -157,13 +157,16 @@ variables () { lxd_service_action () { local action=$1 - if which service > /dev/null 2>&1; then + # upstart + if which service > /dev/null 2>&1 && [ -f "/etc/init/lxd.conf" ] ; then echo "sudo service lxd ${action}" sudo service lxd ${action} - elif which snap > /dev/null 2>&1 && snap connections lxd > /dev/null 2>&1 ; then + # snap + elif snap connections lxd > /dev/null 2>&1 ; then echo "sudo snap ${action} lxd" sudo snap ${action} lxd - elif which systemctl > /dev/null 2>&1 && systemctl cat lxd > /dev/null 2>&1 ; then + # systemd + elif systemctl cat lxd > /dev/null 2>&1 ; then echo "sudo systemctl ${action} lxd" sudo systemctl ${action} lxd else