Dynamically detect number of cores used to build monero_c, since it a…

…ppears that zano requires more memory to link (and it reliably fails for first couple builds due to OOM on CI/VM with memory constrains). Drop unshallowing of all modules [run tests]
cake-tech · Jan 17, 2025 · 5cbf3ea · 5cbf3ea
1 parent c038339
commit 5cbf3ea
Show file tree

Hide file tree

Showing 12 changed files with 51 additions and 42 deletions.
diff --git a/run-android.sh b/run-android.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/universal_sed.sh"
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/functions.sh"
 # Get the current git branch
 get_current_branch() {
     if git rev-parse --git-dir > /dev/null 2>&1; then

diff --git a/scripts/android/build_monero_all.sh b/scripts/android/build_monero_all.sh
@@ -1,13 +1,12 @@
 #!/bin/bash
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/functions.sh"
 
 # Usage: env USE_DOCKER= ./build_all.sh 
 
 set -x -e
 
 cd "$(dirname "$0")"
 
-NPROC="-j$(nproc)"
-
 ../prepare_moneroc.sh
 
 for COIN in monero wownero;
@@ -19,7 +18,7 @@ do
             then
                 echo "file exist, not building monero_c for ${COIN}/$target.";
             else
-                ./build_single.sh ${COIN} $target $NPROC
+                ./build_single.sh ${COIN} $target -j$MAKE_JOB_COUNT
                 unxz -f ../monero_c/release/${COIN}/${target}_libwallet2_api_c.so.xz
             fi
         done

diff --git a/scripts/android/inject_app_details.sh b/scripts/android/inject_app_details.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/universal_sed.sh"
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/functions.sh"
 if [ -z "$APP_ANDROID_TYPE" ]; then
         echo "Please set APP_ANDROID_TYPE"
         exit 1

diff --git a/scripts/functions.sh b/scripts/functions.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+detect_sed() {
+    if sed --version 2>/dev/null | grep -q "GNU"; then
+        SED_TYPE="GNU"
+    else
+        SED_TYPE="BSD"
+    fi
+}
+
+universal_sed() {
+    local expression=$1
+    local file=$2
+
+    if [[ "$SED_TYPE" == "GNU" ]]; then
+        sed -i "$expression" "$file"
+    else
+        sed -i '' "$expression" "$file"
+    fi
+}
+
+detect_sed
+
+if [[ "$(uname)" == "Linux" ]];
+then
+    export MAKE_JOB_COUNT="$(expr $(printf '%s\n%s' $(( $(grep MemTotal: /proc/meminfo | cut -d: -f2 | cut -dk -f1) * 4 / (1048576 * 9) )) $(nproc) | sort -n | head -n1) '|' 1)"
+elif [[ "$(uname)" == "Darwin" ]];
+then
+    export MAKE_JOB_COUNT="(expr $(printf '%s\n%s' $(( $(sysctl -n hw.memsize) * 4 / (1073741824 * 9) )) $(sysctl -n hw.logicalcpu) | sort -n | head -n1) '|' 1)"
+else
+    # Assume windows eh?
+    export MAKE_JOB_COUNT="$(expr $(printf '%s\n%s' $(( $(grep MemTotal: /proc/meminfo | cut -d: -f2 | cut -dk -f1) * 4 / (1048576 * 9) )) $(nproc) | sort -n | head -n1) '|' 1)"
+fi
diff --git a/scripts/ios/app_config.sh b/scripts/ios/app_config.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/universal_sed.sh"
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/functions.sh"
 set -x -e
 MONERO_COM="monero.com"
 CAKEWALLET="cakewallet"

diff --git a/scripts/ios/build_monero_all.sh b/scripts/ios/build_monero_all.sh
@@ -1,4 +1,5 @@
 #!/bin/sh
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/functions.sh"
 
 . ./config.sh
 # ./install_missing_headers.sh
@@ -12,15 +13,13 @@ set -x -e
 
 cd "$(dirname "$0")"
 
-NPROC="-j$(sysctl -n hw.logicalcpu)"
-
 ../prepare_moneroc.sh
 
 for COIN in monero wownero zano;
 do
     pushd ../monero_c
         rm -rf external/ios/build
-        ./build_single.sh ${COIN} aarch64-apple-ios $NPROC
+        ./build_single.sh ${COIN} aarch64-apple-ios -j$MAKE_JOB_COUNT
     popd
 done
 

diff --git a/scripts/linux/app_config.sh b/scripts/linux/app_config.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/universal_sed.sh"
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/functions.sh"
 CAKEWALLET="cakewallet"
 DIR=`pwd`
 

diff --git a/scripts/linux/build_monero_all.sh b/scripts/linux/build_monero_all.sh
@@ -1,11 +1,10 @@
 #!/bin/bash
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/functions.sh"
 
 set -x -e
 
 cd "$(dirname "$0")"
 
-NPROC="-j$(nproc)"
-
 ../prepare_moneroc.sh
 
 for COIN in monero wownero zano;
@@ -17,7 +16,7 @@ do
             then
                 echo "file exist, not building monero_c for ${COIN}/$target.";
             else
-                ./build_single.sh ${COIN} $target $NPROC
+                ./build_single.sh ${COIN} $target -j$MAKE_JOB_COUNT
                 unxz -f ../monero_c/release/${COIN}/${target}_libwallet2_api_c.so.xz
             fi
         done

diff --git a/scripts/macos/app_config.sh b/scripts/macos/app_config.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/universal_sed.sh"
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/functions.sh"
 
 MONERO_COM="monero.com"
 CAKEWALLET="cakewallet"

diff --git a/scripts/macos/gen_common.sh b/scripts/macos/gen_common.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
-source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/universal_sed.sh"
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/functions.sh"
 gen_podspec() {
 	ARCH=$1
 	CW_PLUGIN_DIR="`pwd`/../../cw_monero/macos"

diff --git a/scripts/universal_sed.sh b/scripts/universal_sed.sh
diff --git a/scripts/windows/build_all.sh b/scripts/windows/build_all.sh
@@ -1,4 +1,6 @@
+#!/bin/bash
 set -x -e
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/functions.sh"
 
 cd "$(dirname "$0")"
 
@@ -16,19 +18,18 @@ pushd ../monero_c
     set +e
     command -v sudo && export SUDO=sudo
     set -e
-    NPROC="-j$(nproc)"
     if [[ ! "x$USE_DOCKER" == "x" ]];
     then
         for COIN in monero wownero;
         do
-            $SUDO docker run --platform linux/amd64 -v$HOME/.cache/ccache:/root/.ccache -v$PWD:$PWD -w $PWD --rm -it git.mrcyjanek.net/mrcyjanek/debian:buster bash -c "git config --global --add safe.directory '*'; apt update; apt install -y ccache gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 gperf libtinfo5; ./build_single.sh ${COIN} x86_64-w64-mingw32 $NPROC"
-            # $SUDO docker run --platform linux/amd64 -v$HOME/.cache/ccache:/root/.ccache -v$PWD:$PWD -w $PWD --rm -it git.mrcyjanek.net/mrcyjanek/debian:buster bash -c "git config --global --add safe.directory '*'; apt update; apt install -y ccache gcc-mingw-w64-i686 g++-mingw-w64-i686 gperf libtinfo5; ./build_single.sh ${COIN} i686-w64-mingw32 $NPROC"
+            $SUDO docker run --platform linux/amd64 -v$HOME/.cache/ccache:/root/.ccache -v$PWD:$PWD -w $PWD --rm -it git.mrcyjanek.net/mrcyjanek/debian:buster bash -c "git config --global --add safe.directory '*'; apt update; apt install -y ccache gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 gperf libtinfo5; ./build_single.sh ${COIN} x86_64-w64-mingw32 -j$MAKE_JOB_COUNT"
+            # $SUDO docker run --platform linux/amd64 -v$HOME/.cache/ccache:/root/.ccache -v$PWD:$PWD -w $PWD --rm -it git.mrcyjanek.net/mrcyjanek/debian:buster bash -c "git config --global --add safe.directory '*'; apt update; apt install -y ccache gcc-mingw-w64-i686 g++-mingw-w64-i686 gperf libtinfo5; ./build_single.sh ${COIN} i686-w64-mingw32 -j$MAKE_JOB_COUNT"
         done
     else
         for COIN in monero wownero;
         do
-            $SUDO ./build_single.sh ${COIN} x86_64-w64-mingw32 $NPROC
-            # $SUDO ./build_single.sh ${COIN} i686-w64-mingw32 $NPROC
+            $SUDO ./build_single.sh ${COIN} x86_64-w64-mingw32 -j$MAKE_JOB_COUNT
+            # $SUDO ./build_single.sh ${COIN} i686-w64-mingw32 -j$MAKE_JOB_COUNT
         done
     fi
 popd