Merge branch 'next/minor' of github.com:Start9Labs/start-os into feat…

…/os-welcome

CLEARNET.md

@@ -0,0 +1,40 @@
+# Setting up clearnet for a service interface
+
+NOTE: this guide is for HTTPS only! Other configurations may require a more bespoke setup depending on the service. Please consult the service documentation or the Start9 Community for help with non-HTTPS applications
+
+## Initialize ACME certificate generation
+
+The following command will register your device with an ACME certificate provider, such as letsencrypt
+
+This only needs to be done once.
+
+```
+start-cli net acme init --provider=letsencrypt --contact="mailto:[email protected]"
+```
+
+- `provider` can be `letsencrypt`, `letsencrypt-staging` (useful if you're doing a lot of testing and want to avoid being rate limited), or the url of any provider that supports the [RFC8555](https://datatracker.ietf.org/doc/html/rfc8555) ACME api
+- `contact` can be any valid contact url, typically `mailto:` urls. it can be specified multiple times to set multiple contacts
+
+## Whitelist a domain for ACME certificate acquisition
+
+The following command will tell the OS to use ACME certificates instead of system signed ones for the provided url. In this example, `testing.drbonez.dev`
+
+This must be done for every domain you wish to host on clearnet.
+
+```
+start-cli net acme domain add "testing.drbonez.dev"
+```
+
+## Forward clearnet port
+
+Go into your router settings, and map port 443 on your router to port 5443 on your start-os device. This one port should cover most use cases
+
+## Add domain to service host
+
+The following command will tell the OS to route https requests from the WAN to the provided hostname to the specified service. In this example, we are adding `testing.drbonez.dev` to the host `ui-multi` on the package `hello-world`. To see a list of available host IDs for a given package, run `start-cli package host <PACKAGE> list`
+
+This must be done for every domain you wish to host on clearnet.
+
+```
+start-cli package host hello-world address ui-multi add testing.drbonez.dev
+```

Makefile

@@ -58,7 +58,7 @@ touch:
 metadata: $(VERSION_FILE) $(PLATFORM_FILE) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE)
 
 sudo:
-	sudo -v
+	sudo true
 
 clean:
 	rm -f system-images/**/*.tar
@@ -213,7 +213,7 @@ emulate-reflash: $(ALL_TARGETS)
 	@if [ -z "$(REMOTE)" ]; then >&2 echo "Must specify REMOTE" && false; fi
 	$(call ssh,'sudo /usr/lib/startos/scripts/chroot-and-upgrade --create')
 	$(MAKE) install REMOTE=$(REMOTE) SSHPASS=$(SSHPASS) DESTDIR=/media/startos/next PLATFORM=$(PLATFORM)
-	$(call ssh,'sudo rm -f /media/startos/config/disk.guid')
+	$(call ssh,'sudo rm -f /media/startos/config/disk.guid /media/startos/config/overlay/etc/hostname')
 	$(call ssh,'sudo /media/startos/next/usr/lib/startos/scripts/chroot-and-upgrade --no-sync "apt-get install -y $(shell cat ./build/lib/depends)"')
 
 upload-ota: results/$(BASENAME).squashfs

build/lib/scripts/chroot-and-upgrade

@@ -43,6 +43,8 @@ if [ -z "$NO_SYNC" ]; then
     mount -t overlay \
 		  -olowerdir=/media/startos/current,upperdir=/media/startos/upper/data,workdir=/media/startos/upper/work \
 		  overlay /media/startos/next
+    mkdir -p /media/startos/next/media/startos/root
+    mount --bind /media/startos/root /media/startos/next/media/startos/root
 fi
 
 if [ -n "$ONLY_CREATE" ]; then
@@ -75,6 +77,7 @@ umount /media/startos/next/dev
 umount /media/startos/next/sys
 umount /media/startos/next/proc
 umount /media/startos/next/boot
+umount /media/startos/next/media/startos/root
 
 if [ "$CHROOT_RES" -eq 0 ]; then
 
@@ -84,7 +87,12 @@ if [ "$CHROOT_RES" -eq 0 ]; then
 
     echo 'Upgrading...'
 
-    time mksquashfs /media/startos/next /media/startos/images/next.squashfs -b 4096 -comp gzip
+    if ! time mksquashfs /media/startos/next /media/startos/images/next.squashfs -b 4096 -comp gzip; then
+      umount -R /media/startos/next
+      umount -R /media/startos/upper
+      rm -rf /media/startos/upper /media/startos/next
+      exit 1
+    fi
     hash=$(b3sum /media/startos/images/next.squashfs | head -c 32)
     mv /media/startos/images/next.squashfs /media/startos/images/${hash}.rootfs
     ln -rsf /media/startos/images/${hash}.rootfs /media/startos/config/current.rootfs

build/lib/scripts/prune-images

@@ -33,10 +33,11 @@ if [ -h /media/startos/config/current.rootfs ] && [ -e /media/startos/config/cur
     echo 'Pruning...'
     current="$(readlink -f /media/startos/config/current.rootfs)"
     while [[ "$(df -B1 --output=avail --sync /media/startos/images | tail -n1)" -lt "$needed" ]]; do
-        to_prune="$(ls -t1 /media/startos/images/*.rootfs /media/startos/images/*.squashfs | grep -v "$current" | tail -n1)"
+        to_prune="$(ls -t1 /media/startos/images/*.rootfs /media/startos/images/*.squashfs 2> /dev/null | grep -v "$current" | tail -n1)"
         if [ -e "$to_prune" ]; then
             echo "  Pruning $to_prune"
             rm -rf "$to_prune"
+            sync
         else
             >&2 echo "Not enough space and nothing to prune!"
             exit 1

container-runtime/src/Adapters/EffectCreator.ts

@@ -150,15 +150,15 @@ export function makeEffects(context: EffectContext): Effects {
         stack: new Error().stack,
       }) as ReturnType<T.Effects["bind"]>
     },
-    clearBindings(...[]: Parameters<T.Effects["clearBindings"]>) {
-      return rpcRound("clear-bindings", {}) as ReturnType<
+    clearBindings(...[options]: Parameters<T.Effects["clearBindings"]>) {
+      return rpcRound("clear-bindings", { ...options }) as ReturnType<
         T.Effects["clearBindings"]
       >
     },
     clearServiceInterfaces(
-      ...[]: Parameters<T.Effects["clearServiceInterfaces"]>
+      ...[options]: Parameters<T.Effects["clearServiceInterfaces"]>
     ) {
-      return rpcRound("clear-service-interfaces", {}) as ReturnType<
+      return rpcRound("clear-service-interfaces", { ...options }) as ReturnType<
         T.Effects["clearServiceInterfaces"]
       >
     },
@@ -284,6 +284,9 @@ export function makeEffects(context: EffectContext): Effects {
       >
     },
 
+    getStatus(...[o]: Parameters<T.Effects["getStatus"]>) {
+      return rpcRound("get-status", o) as ReturnType<T.Effects["getStatus"]>
+    },
     setMainStatus(o: { status: "running" | "stopped" }): Promise<null> {
       return rpcRound("set-main-status", o) as ReturnType<
         T.Effects["setHealth"]

container-runtime/src/Adapters/RpcListener.ts

@@ -42,6 +42,7 @@ export const matchRpcResult = anyOf(
     ),
   }),
 )
+
 export type RpcResult = typeof matchRpcResult._TYPE
 type SocketResponse = ({ jsonrpc: "2.0"; id: IdType } & RpcResult) | null
 
@@ -88,7 +89,7 @@ const sandboxRunType = object(
 const callbackType = object({
   method: literal("callback"),
   params: object({
-    callback: number,
+    id: number,
     args: array,
   }),
 })
@@ -135,11 +136,13 @@ const jsonParse = (x: string) => JSON.parse(x)
 
 const handleRpc = (id: IdType, result: Promise<RpcResult>) =>
   result
-    .then((result) => ({
-      jsonrpc,
-      id,
-      ...result,
-    }))
+    .then((result) => {
+      return {
+        jsonrpc,
+        id,
+        ...result,
+      }
+    })
     .then((x) => {
       if (
         ("result" in x && x.result === undefined) ||
@@ -288,8 +291,8 @@ export class RpcListener {
 
         return handleRpc(id, result)
       })
-      .when(callbackType, async ({ params: { callback, args } }) => {
-        this.callCallback(callback, args)
+      .when(callbackType, async ({ params: { id, args } }) => {
+        this.callCallback(id, args)
         return null
       })
       .when(startType, async ({ id }) => {
@@ -410,9 +413,8 @@ export class RpcListener {
     input: any,
   ) {
     const ensureResultTypeShape = (
-      result: void | T.ActionInput | T.PropertiesReturn | T.ActionResult | null,
+      result: void | T.ActionInput | T.ActionResult | null,
     ): { result: any } => {
-      if (isResult(result)) return result
       return { result }
     }
     const callbacks = this.callbackHolderFor(procedure)
@@ -428,8 +430,6 @@ export class RpcListener {
           return system.createBackup(effects, timeout || null)
         case "/backup/restore":
           return system.restoreBackup(effects, timeout || null)
-        case "/properties":
-          return system.properties(effects, timeout || null)
         case "/packageInit":
           return system.packageInit(effects, timeout || null)
         case "/packageUninit":

container-runtime/src/Adapters/Systems/SystemForEmbassy/DockerProcedureContainer.ts

@@ -151,7 +151,7 @@ export class DockerProcedureContainer {
     }
   }
 
-  async spawn(commands: string[]): Promise<cp.ChildProcessWithoutNullStreams> {
+  async spawn(commands: string[]): Promise<cp.ChildProcess> {
     return await this.subcontainer.spawn(commands)
   }
 }

container-runtime/src/Adapters/Systems/SystemForEmbassy/__snapshots__/transformConfigSpec.test.ts.snap

@@ -264,7 +264,6 @@ exports[`transformConfigSpec transformConfigSpec(bitcoind) 1`] = `
         "disabled": false,
         "immutable": false,
         "name": "Pruning Mode",
-        "required": true,
         "type": "union",
         "variants": {
           "automatic": {
@@ -524,7 +523,6 @@ exports[`transformConfigSpec transformConfigSpec(embassyPages) 1`] = `
     "disabled": false,
     "immutable": false,
     "name": "Type",
-    "required": true,
     "type": "union",
     "variants": {
       "index": {
@@ -589,7 +587,6 @@ exports[`transformConfigSpec transformConfigSpec(embassyPages) 1`] = `
             "disabled": false,
             "immutable": false,
             "name": "Folder Location",
-            "required": false,
             "type": "select",
             "values": {
               "filebrowser": "filebrowser",
@@ -644,7 +641,6 @@ exports[`transformConfigSpec transformConfigSpec(embassyPages) 1`] = `
           "disabled": false,
           "immutable": false,
           "name": "Type",
-          "required": true,
           "type": "union",
           "variants": {
             "redirect": {
@@ -705,7 +701,6 @@ exports[`transformConfigSpec transformConfigSpec(embassyPages) 1`] = `
                   "disabled": false,
                   "immutable": false,
                   "name": "Folder Location",
-                  "required": false,
                   "type": "select",
                   "values": {
                     "filebrowser": "filebrowser",
@@ -758,7 +753,6 @@ exports[`transformConfigSpec transformConfigSpec(nostr2) 1`] = `
     "disabled": false,
     "immutable": false,
     "name": "Relay Type",
-    "required": true,
     "type": "union",
     "variants": {
       "private": {