Merge branch 'next/minor' of github.com:Start9Labs/start-os into feat…

…ure/wireguard-and-firewall

.github/workflows/startos-iso.yaml

@@ -78,7 +78,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.x'
+          python-version: "3.x"
 
       - uses: actions/setup-node@v4
         with:
@@ -156,7 +156,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.x'
+          python-version: "3.x"
 
       - name: Install dependencies
         run: |
@@ -187,11 +187,27 @@ jobs:
         run: |
           mkdir -p web/node_modules
           mkdir -p web/dist/raw
-          touch core/startos/bindings
-          touch sdk/lib/osBindings
+          mkdir -p core/startos/bindings
+          mkdir -p sdk/base/lib/osBindings
+          mkdir -p container-runtime/node_modules
           mkdir -p container-runtime/dist
+          mkdir -p container-runtime/dist/node_modules
+          mkdir -p core/startos/bindings
+          mkdir -p sdk/dist
+          mkdir -p sdk/baseDist
+          mkdir -p patch-db/client/node_modules
+          mkdir -p patch-db/client/dist
+          mkdir -p web/.angular
+          mkdir -p web/dist/raw/ui
+          mkdir -p web/dist/raw/install-wizard
+          mkdir -p web/dist/raw/setup-wizard
+          mkdir -p web/dist/static/ui
+          mkdir -p web/dist/static/install-wizard
+          mkdir -p web/dist/static/setup-wizard
           PLATFORM=${{ matrix.platform }} make -t compiled-${{ env.ARCH }}.tar
 
+      - run: git status
+
       - name: Run iso build
         run: PLATFORM=${{ matrix.platform }} make iso
         if: ${{ matrix.platform != 'raspberrypi' }}

.github/workflows/test.yaml

@@ -11,7 +11,7 @@ on:
       - next/*
 
 env:
-  NODEJS_VERSION: "18.15.0"
+  NODEJS_VERSION: "20.16.0"
   ENVIRONMENT: dev-unstable
 
 jobs:

CLEARNET.md

@@ -0,0 +1,40 @@
+# Setting up clearnet for a service interface
+
+NOTE: this guide is for HTTPS only! Other configurations may require a more bespoke setup depending on the service. Please consult the service documentation or the Start9 Community for help with non-HTTPS applications
+
+## Initialize ACME certificate generation
+
+The following command will register your device with an ACME certificate provider, such as letsencrypt
+
+This only needs to be done once.
+
+```
+start-cli net acme init --provider=letsencrypt --contact="mailto:[email protected]"
+```
+
+- `provider` can be `letsencrypt`, `letsencrypt-staging` (useful if you're doing a lot of testing and want to avoid being rate limited), or the url of any provider that supports the [RFC8555](https://datatracker.ietf.org/doc/html/rfc8555) ACME api
+- `contact` can be any valid contact url, typically `mailto:` urls. it can be specified multiple times to set multiple contacts
+
+## Whitelist a domain for ACME certificate acquisition
+
+The following command will tell the OS to use ACME certificates instead of system signed ones for the provided url. In this example, `testing.drbonez.dev`
+
+This must be done for every domain you wish to host on clearnet.
+
+```
+start-cli net acme domain add "testing.drbonez.dev"
+```
+
+## Forward clearnet port
+
+Go into your router settings, and map port 443 on your router to port 5443 on your start-os device. This one port should cover most use cases
+
+## Add domain to service host
+
+The following command will tell the OS to route https requests from the WAN to the provided hostname to the specified service. In this example, we are adding `testing.drbonez.dev` to the host `ui-multi` on the package `hello-world`. To see a list of available host IDs for a given package, run `start-cli package host <PACKAGE> list`
+
+This must be done for every domain you wish to host on clearnet.
+
+```
+start-cli package host hello-world address ui-multi add testing.drbonez.dev
+```

DEVELOPMENT.md

@@ -27,6 +27,7 @@ curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/master/install.sh | bash
 source ~/.bashrc
 nvm install 20
 nvm use 20
+nvm alias default 20 # this prevents your machine from reverting back to another version
 ```
 
 ## Cloning the repository

Makefile

@@ -6,7 +6,8 @@ BASENAME := $(shell ./basename.sh)
 PLATFORM := $(shell if [ -f ./PLATFORM.txt ]; then cat ./PLATFORM.txt; else echo unknown; fi)
 ARCH := $(shell if [ "$(PLATFORM)" = "raspberrypi" ]; then echo aarch64; else echo $(PLATFORM) | sed 's/-nonfree$$//g'; fi)
 IMAGE_TYPE=$(shell if [ "$(PLATFORM)" = raspberrypi ]; then echo img; else echo iso; fi)
-WEB_UIS := web/dist/raw/ui web/dist/raw/setup-wizard web/dist/raw/install-wizard
+WEB_UIS := web/dist/raw/ui/index.html web/dist/raw/setup-wizard/index.html web/dist/raw/install-wizard/index.html
+COMPRESSED_WEB_UIS := web/dist/static/ui/index.html web/dist/static/setup-wizard/index.html web/dist/static/install-wizard/index.html
 FIRMWARE_ROMS := ./firmware/$(PLATFORM) $(shell jq --raw-output '.[] | select(.platform[] | contains("$(PLATFORM)")) | "./firmware/$(PLATFORM)/" + .id + ".rom.gz"' build/lib/firmware.json)
 BUILD_SRC := $(shell git ls-files build) build/lib/depends build/lib/conflicts $(FIRMWARE_ROMS)
 DEBIAN_SRC := $(shell git ls-files debian/)
@@ -16,7 +17,7 @@ COMPAT_SRC := $(shell git ls-files system-images/compat/)
 UTILS_SRC := $(shell git ls-files system-images/utils/)
 BINFMT_SRC := $(shell git ls-files system-images/binfmt/)
 CORE_SRC := $(shell git ls-files core) $(shell git ls-files --recurse-submodules patch-db) $(GIT_HASH_FILE)
-WEB_SHARED_SRC := $(shell git ls-files web/projects/shared) $(shell ls -p web/ | grep -v / | sed 's/^/web\//g') web/node_modules/.package-lock.json web/config.json patch-db/client/dist web/patchdb-ui-seed.json sdk/dist
+WEB_SHARED_SRC := $(shell git ls-files web/projects/shared) $(shell ls -p web/ | grep -v / | sed 's/^/web\//g') web/node_modules/.package-lock.json web/config.json patch-db/client/dist/index.js sdk/baseDist/package.json web/patchdb-ui-seed.json sdk/dist/package.json
 WEB_UI_SRC := $(shell git ls-files web/projects/ui)
 WEB_SETUP_WIZARD_SRC := $(shell git ls-files web/projects/setup-wizard)
 WEB_INSTALL_WIZARD_SRC := $(shell git ls-files web/projects/install-wizard)
@@ -47,7 +48,7 @@ endif
 
 .DELETE_ON_ERROR:
 
-.PHONY: all metadata install clean format cli uis ui reflash deb $(IMAGE_TYPE) squashfs sudo wormhole wormhole-deb test test-core test-sdk test-container-runtime
+.PHONY: all metadata install clean format cli uis ui reflash deb $(IMAGE_TYPE) squashfs sudo wormhole wormhole-deb test test-core test-sdk test-container-runtime registry
 
 all: $(ALL_TARGETS)
 
@@ -92,17 +93,20 @@ format:
 test: | test-core test-sdk test-container-runtime
 
 test-core: $(CORE_SRC) $(ENVIRONMENT_FILE) 
-	cd core && cargo build --features=test && cargo test --features=test
+	./core/run-tests.sh
 
-test-sdk: $(shell git ls-files sdk) sdk/lib/osBindings
+test-sdk: $(shell git ls-files sdk) sdk/base/lib/osBindings/index.ts
 	cd sdk && make test
 
-test-container-runtime: container-runtime/node_modules $(shell git ls-files container-runtime/src) container-runtime/package.json container-runtime/tsconfig.json 
+test-container-runtime: container-runtime/node_modules/.package-lock.json $(shell git ls-files container-runtime/src) container-runtime/package.json container-runtime/tsconfig.json 
 	cd container-runtime && npm test
 
 cli:
 	cd core && ./install-cli.sh
 
+registry:
+	cd core && ./build-registrybox.sh
+
 deb: results/$(BASENAME).deb
 
 debian/control: build/lib/depends build/lib/conflicts
@@ -209,7 +213,7 @@ emulate-reflash: $(ALL_TARGETS)
 	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
 	$(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create')
 	$(MAKE) install REMOTE=$(REMOTE) SSHPASS=$(SSHPASS) DESTDIR=/media/startos/next PLATFORM=$(PLATFORM)
-	$(call ssh,'sudo rm -f /media/startos/config/disk.guid')
+	$(call ssh,'sudo rm -f /media/startos/config/disk.guid /media/startos/config/overlay/etc/hostname')
 	$(call ssh,'sudo /media/startos/next/usr/lib/startos/scripts/chroot-and-upgrade --no-sync "apt-get install -y $(shell cat ./build/lib/depends)"')
 
 upload-ota: results/$(BASENAME).squashfs
@@ -218,34 +222,36 @@ upload-ota: results/$(BASENAME).squashfs
 container-runtime/debian.$(ARCH).squashfs:
 	ARCH=$(ARCH) ./container-runtime/download-base-image.sh
 
-container-runtime/node_modules: container-runtime/package.json container-runtime/package-lock.json sdk/dist
+container-runtime/node_modules/.package-lock.json: container-runtime/package.json container-runtime/package-lock.json sdk/dist/package.json
 	npm --prefix container-runtime ci
-	touch container-runtime/node_modules
+	touch container-runtime/node_modules/.package-lock.json
 
-sdk/lib/osBindings: core/startos/bindings
-	mkdir -p sdk/lib/osBindings
-	ls core/startos/bindings/*.ts | sed 's/core\/startos\/bindings\/\([^.]*\)\.ts/export { \1 } from ".\/\1";/g' > core/startos/bindings/index.ts
-	npm --prefix sdk exec -- prettier --config ./sdk/package.json -w ./core/startos/bindings/*.ts
-	rsync -ac --delete core/startos/bindings/ sdk/lib/osBindings/
-	touch sdk/lib/osBindings
+sdk/base/lib/osBindings/index.ts: core/startos/bindings/index.ts
+	mkdir -p sdk/base/lib/osBindings
+	rsync -ac --delete core/startos/bindings/ sdk/base/lib/osBindings/
+	touch sdk/base/lib/osBindings/index.ts
 
-core/startos/bindings: $(shell git ls-files core) $(ENVIRONMENT_FILE)
+core/startos/bindings/index.ts: $(shell git ls-files core) $(ENVIRONMENT_FILE)
 	rm -rf core/startos/bindings
-	(cd core/ && cargo test --features=test 'export_bindings_')
-	touch core/startos/bindings
+	./core/build-ts.sh
+	ls core/startos/bindings/*.ts | sed 's/core\/startos\/bindings\/\([^.]*\)\.ts/export { \1 } from ".\/\1";/g' | grep -v '"./index"' | tee core/startos/bindings/index.ts
+	npm --prefix sdk exec -- prettier --config ./sdk/base/package.json -w ./core/startos/bindings/*.ts
+	touch core/startos/bindings/index.ts
 
-sdk/dist: $(shell git ls-files sdk) sdk/lib/osBindings
+sdk/dist/package.json sdk/baseDist/package.json: $(shell git ls-files sdk) sdk/base/lib/osBindings/index.ts
 	(cd sdk && make bundle)
+	touch sdk/dist/package.json
+	touch sdk/baseDist/package.json
 
 # TODO: make container-runtime its own makefile?
-container-runtime/dist/index.js: container-runtime/node_modules $(shell git ls-files container-runtime/src) container-runtime/package.json container-runtime/tsconfig.json 
+container-runtime/dist/index.js: container-runtime/node_modules/.package-lock.json $(shell git ls-files container-runtime/src) container-runtime/package.json container-runtime/tsconfig.json 
 	npm --prefix container-runtime run build
 
-container-runtime/dist/node_modules container-runtime/dist/package.json container-runtime/dist/package-lock.json: container-runtime/package.json container-runtime/package-lock.json sdk/dist container-runtime/install-dist-deps.sh
+container-runtime/dist/node_modules/.package-lock.json container-runtime/dist/package.json container-runtime/dist/package-lock.json: container-runtime/package.json container-runtime/package-lock.json sdk/dist/package.json container-runtime/install-dist-deps.sh
 	./container-runtime/install-dist-deps.sh
-	touch container-runtime/dist/node_modules
+	touch container-runtime/dist/node_modules/.package-lock.json
 
-container-runtime/rootfs.$(ARCH).squashfs: container-runtime/debian.$(ARCH).squashfs container-runtime/container-runtime.service container-runtime/update-image.sh container-runtime/deb-install.sh container-runtime/dist/index.js container-runtime/dist/node_modules core/target/$(ARCH)-unknown-linux-musl/release/containerbox | sudo
+container-runtime/rootfs.$(ARCH).squashfs: container-runtime/debian.$(ARCH).squashfs container-runtime/container-runtime.service container-runtime/update-image.sh container-runtime/deb-install.sh container-runtime/dist/index.js container-runtime/dist/node_modules/.package-lock.json core/target/$(ARCH)-unknown-linux-musl/release/containerbox | sudo
 	ARCH=$(ARCH) ./container-runtime/update-image.sh
 
 build/lib/depends build/lib/conflicts: build/dpkg-deps/*
@@ -263,35 +269,36 @@ system-images/utils/docker-images/$(ARCH).tar: $(UTILS_SRC)
 system-images/binfmt/docker-images/$(ARCH).tar: $(BINFMT_SRC)
 	cd system-images/binfmt && make docker-images/$(ARCH).tar && touch docker-images/$(ARCH).tar
 
-core/target/$(ARCH)-unknown-linux-musl/release/startbox: $(CORE_SRC) web/dist/static web/patchdb-ui-seed.json $(ENVIRONMENT_FILE)
+core/target/$(ARCH)-unknown-linux-musl/release/startbox: $(CORE_SRC) $(COMPRESSED_WEB_UIS) web/patchdb-ui-seed.json $(ENVIRONMENT_FILE)
 	ARCH=$(ARCH) ./core/build-startbox.sh
 	touch core/target/$(ARCH)-unknown-linux-musl/release/startbox
 
 core/target/$(ARCH)-unknown-linux-musl/release/containerbox: $(CORE_SRC) $(ENVIRONMENT_FILE)
 	ARCH=$(ARCH) ./core/build-containerbox.sh
 	touch core/target/$(ARCH)-unknown-linux-musl/release/containerbox
 
-web/node_modules/.package-lock.json: web/package.json sdk/dist
+web/node_modules/.package-lock.json: web/package.json sdk/baseDist/package.json
 	npm --prefix web ci
 	touch web/node_modules/.package-lock.json
 
-web/.angular: patch-db/client/dist sdk/dist web/node_modules/.package-lock.json
+web/.angular/.updated: patch-db/client/dist/index.js sdk/baseDist/package.json web/node_modules/.package-lock.json
 	rm -rf web/.angular
 	mkdir -p web/.angular
+	touch web/.angular/.updated
 
-web/dist/raw/ui: $(WEB_UI_SRC) $(WEB_SHARED_SRC) web/.angular
+web/dist/raw/ui/index.html: $(WEB_UI_SRC) $(WEB_SHARED_SRC) web/.angular/.updated
 	npm --prefix web run build:ui
-	touch web/dist/raw/ui
+	touch web/dist/raw/ui/index.html
 
-web/dist/raw/setup-wizard: $(WEB_SETUP_WIZARD_SRC) $(WEB_SHARED_SRC) web/.angular
+web/dist/raw/setup-wizard/index.html: $(WEB_SETUP_WIZARD_SRC) $(WEB_SHARED_SRC) web/.angular/.updated
 	npm --prefix web run build:setup
-	touch web/dist/raw/setup-wizard
+	touch web/dist/raw/setup-wizard/index.html
 
-web/dist/raw/install-wizard: $(WEB_INSTALL_WIZARD_SRC) $(WEB_SHARED_SRC) web/.angular
+web/dist/raw/install-wizard/index.html: $(WEB_INSTALL_WIZARD_SRC) $(WEB_SHARED_SRC) web/.angular/.updated
 	npm --prefix web run build:install-wiz
-	touch web/dist/raw/install-wizard
+	touch web/dist/raw/install-wizard/index.html
 
-web/dist/static: $(WEB_UIS) $(ENVIRONMENT_FILE)
+$(COMPRESSED_WEB_UIS): $(WEB_UIS) $(ENVIRONMENT_FILE)
 	./compress-uis.sh
 
 web/config.json: $(GIT_HASH_FILE) web/config-sample.json
@@ -301,13 +308,14 @@ web/patchdb-ui-seed.json: web/package.json
 	jq '."ack-welcome" = $(shell jq '.version' web/package.json)' web/patchdb-ui-seed.json > ui-seed.tmp
 	mv ui-seed.tmp web/patchdb-ui-seed.json
 
-patch-db/client/node_modules: patch-db/client/package.json
+patch-db/client/node_modules/.package-lock.json: patch-db/client/package.json
 	npm --prefix patch-db/client ci
-	touch patch-db/client/node_modules
+	touch patch-db/client/node_modules/.package-lock.json
 
-patch-db/client/dist: $(PATCH_DB_CLIENT_SRC) patch-db/client/node_modules
+patch-db/client/dist/index.js: $(PATCH_DB_CLIENT_SRC) patch-db/client/node_modules/.package-lock.json
 	rm -rf patch-db/client/dist
 	npm --prefix patch-db/client run build
+	touch patch-db/client/dist/index.js
 
 # used by github actions
 compiled-$(ARCH).tar: $(COMPILED_TARGETS) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) $(VERSION_FILE)
@@ -326,4 +334,4 @@ cargo-deps/$(ARCH)-unknown-linux-musl/release/tokio-console:
 	ARCH=$(ARCH) PREINSTALL="apk add musl-dev pkgconfig" ./build-cargo-dep.sh tokio-console
 
 cargo-deps/$(ARCH)-unknown-linux-musl/release/startos-backup-fs:
-	ARCH=$(ARCH) PREINSTALL="apk add fuse3 fuse3-dev fuse3-static musl-dev pkgconfig" ./build-cargo-dep.sh --git https://github.com/Start9Labs/start-fs.git startos-backup-fs
+	ARCH=$(ARCH) PREINSTALL="apk add fuse3 fuse3-dev fuse3-static musl-dev pkgconfig" ./build-cargo-dep.sh --git https://github.com/Start9Labs/start-fs.git startos-backup-fs

build-cargo-dep.sh

@@ -18,7 +18,7 @@ if [ -z "$ARCH" ]; then
 fi
 
 DOCKER_PLATFORM="linux/${ARCH}"
-if [ "$ARCH" = aarch64 ]; then
+if [ "$ARCH" = aarch64 ] || [ "$ARCH" = arm64 ]; then
 	DOCKER_PLATFORM="linux/arm64"
 elif [ "$ARCH" = x86_64 ]; then
 	DOCKER_PLATFORM="linux/amd64"

build/dpkg-deps/depends

@@ -9,6 +9,7 @@ ca-certificates
 cifs-utils
 cryptsetup
 curl
+dnsutils
 dmidecode
 dnsutils
 dosfstools
@@ -48,6 +49,7 @@ smartmontools
 socat
 sqlite3
 squashfs-tools
+squashfs-tools-ng
 sudo
 systemd
 systemd-resolved

build/lib/scripts/chroot-and-upgrade

@@ -43,6 +43,8 @@ if [ -z "$NO_SYNC" ]; then
     mount -t overlay \
 		  -olowerdir=/media/startos/current,upperdir=/media/startos/upper/data,workdir=/media/startos/upper/work \
 		  overlay /media/startos/next
+    mkdir -p /media/startos/next/media/startos/root
+    mount --bind /media/startos/root /media/startos/next/media/startos/root
 fi
 
 if [ -n "$ONLY_CREATE" ]; then
@@ -75,6 +77,7 @@ umount /media/startos/next/dev
 umount /media/startos/next/sys
 umount /media/startos/next/proc
 umount /media/startos/next/boot
+umount /media/startos/next/media/startos/root
 
 if [ "$CHROOT_RES" -eq 0 ]; then
 
@@ -84,7 +87,12 @@ if [ "$CHROOT_RES" -eq 0 ]; then
 
     echo 'Upgrading...'
 
-    time mksquashfs /media/startos/next /media/startos/images/next.squashfs -b 4096 -comp gzip
+    if ! time mksquashfs /media/startos/next /media/startos/images/next.squashfs -b 4096 -comp gzip; then
+      umount -R /media/startos/next
+      umount -R /media/startos/upper
+      rm -rf /media/startos/upper /media/startos/next
+      exit 1
+    fi
     hash=$(b3sum /media/startos/images/next.squashfs | head -c 32)
     mv /media/startos/images/next.squashfs /media/startos/images/${hash}.rootfs
     ln -rsf /media/startos/images/${hash}.rootfs /media/startos/config/current.rootfs

build/lib/scripts/prune-images

@@ -33,10 +33,11 @@ if [ -h /media/startos/config/current.rootfs ] && [ -e /media/startos/config/cur
     echo 'Pruning...'
     current="$(readlink -f /media/startos/config/current.rootfs)"
     while [[ "$(df -B1 --output=avail --sync /media/startos/images | tail -n1)" -lt "$needed" ]]; do
-        to_prune="$(ls -t1 /media/startos/images/*.rootfs /media/startos/images/*.squashfs | grep -v "$current" | tail -n1)"
+        to_prune="$(ls -t1 /media/startos/images/*.rootfs /media/startos/images/*.squashfs 2> /dev/null | grep -v "$current" | tail -n1)"
         if [ -e "$to_prune" ]; then
             echo "  Pruning $to_prune"
             rm -rf "$to_prune"
+            sync
         else
             >&2 echo "Not enough space and nothing to prune!"
             exit 1