Output and Authentication upgrade (#17)

* update readme and add compose plugin

* fix compose location

* disable compose integration into rootfs, binary size increase is excessive

* implement journal, wip TOPT authentication mode

* working totp implementation

* static checks

* update image

* correct naming

* add error to hotp

Dockerfile

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-FROM alpine:3.14.0
+FROM alpine:3.14.1
 
 RUN apk --no-cache add \
     alpine-sdk argp-standalone asciidoc autoconf automake bc bison build-base ccache clang cmake cryptsetup coreutils \
@@ -24,14 +24,15 @@ RUN mkdir -p ${OUT_DIR} && mkdir -p ${SRC_DIR} && mkdir -p /rootfs
 WORKDIR ${SRC_DIR}
 
 # Package versions
-ENV VERSION_KERNEL=5.10.41
-ENV VERSION_RT=5.10.41-rt42
+ENV VERSION_KERNEL=5.10.65
+ENV VERSION_RT=5.10.65-rt53
 ENV VERSION_MUSL=1.2.2
 ENV VERSION_DOCKER=20.10.7
 ENV VERSION_BUSYBOX=1.33.1
 ENV VERSION_WGTOOLS=v1.0.20210424
 ENV VERSION_MICROCODE_INTEL=20210608
 ENV VERSION_IPTABLES=1.8.7
+ENV VERSION_COMPOSE=2.0.0-rc.3
 
 # Flags
 ENV CONFIG_KERNEL=5.10.1-rt20
@@ -47,8 +48,8 @@ RUN wget -q -O busybox.tar.bz2 https://busybox.net/downloads/busybox-${VERSION_B
 RUN wget -q -O microcode.tar.gz https://github.com/intel/Intel-Linux-Processor-Microcode-Data-Files/archive/refs/tags/microcode-${VERSION_MICROCODE_INTEL}.tar.gz 
 
 # Verify sources
-RUN echo "f604759de80767c4f8bdc500eec730dc161bc914a48bd366b748c176701a6771 kernel.tar.xz" | sha256sum -c -
-RUN echo "03a1be966680c3fc8853d8b1d08fca3dd1303961e471d5bb41e44d57b07e12fd patch-rt.xz" | sha256sum -c -
+RUN echo "edd3dedbce5bcaa5ba7cde62f8f3fd58b2ab21e2ec427b9d200685da5ec03e66 kernel.tar.xz" | sha256sum -c -
+RUN echo "a355068c2802a52705f00c0a61afc73ced4ecb8d712976fa80ac9584068bbeeb patch-rt.xz" | sha256sum -c -
 RUN echo "9b969322012d796dc23dda27a35866034fa67d8fb67e0e2c45c913c3d43219dd musl.tar.gz" | sha256sum -c -
 RUN echo "34ad50146fce29b28e5115a1e8510dd5232459c9a4a9f28f65909f92cca314d9 docker.tgz" | sha256sum -c -
 RUN echo "98140aa91ea04018ebd874c14ab9b6994f48cdaf9a219ccf7c0cd3e513c7428a wireguard.tar.xz" | sha256sum -c -
@@ -109,12 +110,12 @@ RUN cp wireguard-tools-${VERSION_WGTOOLS}/src/wg /rootfs/usr/sbin/wg
 RUN cp docker/* /rootfs/usr/bin/
 
 # Add alpine packages
-RUN cd /bin && cp -t /rootfs/bin lsblk 
+RUN cd /bin && cp -t /rootfs/bin lsblk
 RUN cd /lib && cp -t /rootfs/lib libblkid.so.* libsmartcols.so.* libmount.so.*
 
 # Strip modules if specified
-ARG CONFIG_MODULES=ALL
-COPY config/modules .
+ARG CONFIG_MODULES=standard.mod
+COPY config/modules/${CONFIG_MODULES} .
 RUN find /rootfs/lib/modules | grep "\.ko$" > ${OUT_DIR}/modules.txt
 RUN if [ "${CONFIG_MODULES}" != "ALL" ]; then find /rootfs/lib/modules | grep "\.ko$" | grep -v -f ${CONFIG_MODULES} | xargs rm; fi;
 RUN find /rootfs/lib/modules | grep "\.ko$" > ${OUT_DIR}/modules_selected.txt
@@ -136,9 +137,10 @@ COPY init init
 RUN go build -ldflags "-s -w" -o /rootfs/init ./init && strip /rootfs/init
 
 # Copy in primary config, and default secondary config to rootfs
-ARG CONFIG_PRIMARY=CONFIG_PRIMARY_UNSET
+ARG CONFIG_PRIMARY=default.yml
 COPY config/primary/$CONFIG_PRIMARY /rootfs/config/primary.yml
-COPY config/secondary/default.yml /rootfs/config/secondary.yml
+ARG CONFIG_SECONDARY=default.yml
+COPY config/secondary/$CONFIG_SECONDARY /rootfs/config/default_secondary.yml
 RUN find /rootfs > ${OUT_DIR}/rootfs.txt
 
 # Build initramfs

Makefile

@@ -3,31 +3,25 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-default: 	standard
+IMAGE = containers.fouwels.app/fouwels/os-next
+TAG = local
+
+build: standard
+
+standard: 	# Standard build configuration
+	docker build -t $(IMAGE):$(TAG) .
+
+fast: 		# fast target for development/qemu
+	docker build --build-arg COMPRESSION_LEVEL=9 -t $(IMAGE):$(TAG) .
+
+fat: 		# fat target with all modules packed and available to be loaded
+	docker build --build-arg CONFIG_MODULES=ALL -t $(IMAGE):$(TAG) .
+
 DRPC-230: 	standard # IMI DRPC-230 target
 k300: 		standard # OnLogic K300 target
 k700: 		standard # OnLogic K700 target
 magellis: 	standard # Schneider Magellis target
-
-standard: # Standard build configuration
-	docker build \
-	--build-arg CONFIG_PRIMARY=standard.yml \
-	--build-arg CONFIG_MODULES=standard.mod \
-	-t containers.fouwels.app/fouwels/os-next:local .
-
-turbo: # fast target for development/qemu
-	docker build \
-	--build-arg COMPRESSION_LEVEL=9 \
-	--build-arg CONFIG_PRIMARY=standard.yml \
-	--build-arg CONFIG_MODULES=standard.mod \
-	-t containers.fouwels.app/fouwels/os-next:local .
-
-all: # Generic fat target with all modules available to be loaded
-	docker build \
-	--build-arg CONFIG_PRIMARY=standard.yml \
-	--build-arg CONFIG_MODULES=ALL \
-	-t containers.fouwels.app/fouwels/os-next:local .
 
 run:
 	docker container rm os-builder || true
-	docker run -it --name os-builder -v $(PWD)/out:/out containers.fouwels.app/fouwels/os-next:local
+	docker run -it --name os-builder -v $(PWD)/out:/out $(IMAGE):$(TAG)

README.md

@@ -6,13 +6,43 @@ SPDX-License-Identifier: Apache-2.0
 -->
 
 # os-next
-This is a real-time IIoT OS made for Docker. All is build from within a container using musl rather than libc. The Dockerfile included is used to create the toolchain image. 
 
-It is a EFI based boot system.
+Minimal, declarative, go based real-time capable linux runtime for deployment of containers at the edge, all in < 45MB.
 
-## Quick Start
+Built against linux with the PREEMPT_RT patch set, with muslc and GCC.
 
-In the `build.sh` script use the KERNEL_CONFIG variable to set the kernel config to be used in the compilation 
+Built as a single EFI, allowing fully declarative builds with one click "firmware" deployment, audit, and upgrade.
+
+EFI based boot system, booted directly from UEFI, no bootloader required (or wanted).
+
+Includes QEMU build target for testing/development. see `deploy/qemu`.
+
+KConfig kernel configuration is derived from Alpine Linux, tailored for RT and specific module requirements.
+
+Generic build target has been developed and deployed for:
+- [OnLogic Karbon 700](https://www.onlogic.com/k300/)
+- [OnLogic Karbon 300](https://www.onlogic.com/k700/)
+- [Scheider Magellis](https://www.se.com/uk/en/product/HMIBMUSI29D2801/modular-box-pc-hmibm-universal-ssd-dc-windows-10-2-slots/)
+- [IEI DRPC-230](https://www.ieiworld.com/en/product/model.php?II=714)
+- [Hyper-V](https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/about/)
+
+Additional platforms may require additional kernel modules specified to be built and loaded via the primary config (see BUILDING), for full IO functionality. Specific device targets were consolidated to prevent excessive divergence, at the expense of a minor increase in binary size.
+
+![init](./docs/img/init.png "Init")
+
+## Building
+
+Build has been converted to build internally with docker/buildkit, full build process is defined in `Dockerfile`. legacy `build.sh` should not be used.
+
+Full source build including kernel compression is runs in ~240 seconds with dual Xeon E5-2670 v3 @ 2.30 GHz. Limiting build factor is the final kernel/initramfs compression, ZSTD parallelism decays signficantly at higher ratios. Fast target will build from scratch in ~45 seconds at the expense of final binary size. 
+
+Cached fast builds for development can be significantly faster (~15 seconds), recommended workflow for a full kernel build is to re-build the fast target, and run the qemu deploy target - see `deploy/qemu`.
+
+![buildkit build](./docs/img/buildkit.png "Buildkit")
+
+Build will fetch, verify, and build sources, build go/uinit, before assembling into a linux build, and rootfs/initramfs, packed into a single distributable EFI.
+
+EFI should be installed to a fat32 formatted partition 0 on a GPT disk, as `EFI/BOOT/BOOTx64.EFI`. This will allow booting via UEFI without an intermediary boot loader. Kernel parameters should be configured in the UEFI boot entry if required.
 
 Ensure docker is compiling with buildkit for a layer based build log (highly recommended)
 
@@ -22,37 +52,38 @@ Build the kernel EFI (`make <target> # make k300|schneider|...`)
 
     docker build -t <build arguments> os-next
 
-The following build arguments are specified.
+The following build arguments are available.
 
-_Included for documentation, see Makefile for existing targets, makefile should be used instead of direct calling_
+_Included for documentation, see Makefile for existing targets, target should be added to makefile instead of direct calling_
 
-    --build-arg CONFIG_PRIMARY=standard.yml # One of config/primary
-    --build-arg CONFIG_MODULES=ALL # ALL or One of config/modules
-    --build-arg COMPRESSION_LEVEL=9 # (optionally) override the default kernel ZSTD compression level (9 for fast, 22 for maximum)
+    --build-arg CONFIG_PRIMARY=standard.yml # Override l1 config, one of config/primary - baked into the EFI
+    --build-arg CONFIG_PRIMARY=secondary.yml # Override l2 config, of config/secondary - modifyable, and copied to the filesystem at runtime, if not existing.
+    --build-arg CONFIG_MODULES=ALL # Override module set, ALL, or one of config/modules - defines modules compiled and shipped with EFI, to allow loading via primary/secondary config. Define as ALL to ship all modules, at the expensive of a huge build
+    --build-arg COMPRESSION_LEVEL=9 # Override the default kernel ZSTD compression level (9 for fast, 22 for maximum)
 
 Copy the kernel EFI from the built image to ./out (`make run`)
 
     docker run -it --rm -v $(PWD)/out:/out os-next
 
-The dockerfile is constructed to cache layers between builds, if required files have not been modified. There is no need to preserve files on a volume.
+The dockerfile is constructed to cache layers between builds, if required files have not been modified. There is no need to preserve files on a volume (anymore)
 
-## Deploying the EFI image
+## Deployment
 
 The image is copied out of the container when the build is successfully completed and place in a subdirectory called `out` in current working director. The file is called BOOTx64.EFI
 
 To deploy this onto a physical hardware device, this hardware needs to support UEFI boot, which is a modern boot loader, which is suported in most new BIOS implementaitons. 
 
 To install, format a drive with the labels and filesystems stated in the primary config.
 
-Then create a directory structure on this device <partition 1>/EFI/BOOT and simply copy the BOOTx64.EFI into the folder. The UEFI will identify this path and boot.
+Then create a directory structure on this device <partition 1>/EFI/BOOT and copy the BOOTx64.EFI into the folder. The UEFI will identify this path and boot.
 
 The OS will use the LABEL fields to map devices to mount points specified in the primary config. Device IDs (eg. /dev/sda1) are not used to allow common operation across disk types (eg. /dev/nvme0n1) 
 
 See `deploy/qemu` for a software deployment. This will automatically format and set up a drive, before starting with QEMU/KVM.
 
 Unless specified, run `make deploy-clean` within, to build and start the kernel and connect stdin/stdout over virtual IO.
 
-## TIPS and TRICKS
+## Info
 
 In the container you can find the linux source directory under /build/src/linux... 
 
@@ -61,3 +92,12 @@ Use `menuconfig` to setup the kernel then copy the .config to /build/out
     `cp .config /build/out`
 
 This will copy the .config to the host machine.
+
+## CI
+CI will build, verify, scan, and statically check the build and uinit. The use of docker/buildkit allows build dependencies to be scanned at build and integration time. See `.github/workflows` for specific steps.
+
+## License
+Project is licenced as Apache 2.0, refer to SPDX tags for specific file based licensing and copyright attribution. All contributed code must be Apache 2.0 compatible. (A)GPL-2.0 and (A)GPL-3.0 is not to be included, and will be rejected by CI.
+
+## History
+Developed within Lagoni Engineering, later acquired by Belcan International. Licensed as open source and continued in development by the original authors.

config/config-5.10.1-rt20

@@ -2360,7 +2360,7 @@ CONFIG_LOCKD=m
 CONFIG_LOCKD_V4=y
 CONFIG_LOCK_SPIN_ON_OWNER=y
 CONFIG_LOCK_DEBUGGING_SUPPORT=y
-CONFIG_LOCALVERSION="-phoenix"
+CONFIG_LOCALVERSION="os-next"
 CONFIG_LOCALVERSION_AUTO=y
 CONFIG_LLD_VERSION=0
 CONFIG_LLC2=m
@@ -3599,7 +3599,7 @@ CONFIG_DEFAULT_TCP_CONG="cubic"
 CONFIG_DEFAULT_SECURITY_DAC=y
 CONFIG_DEFAULT_MMAP_MIN_ADDR=4096
 CONFIG_DEFAULT_INIT=""
-CONFIG_DEFAULT_HOSTNAME="phoenix"
+CONFIG_DEFAULT_HOSTNAME="os-next"
 CONFIG_DEFAULT_CUBIC=y
 CONFIG_DECOMPRESS_ZSTD=y
 CONFIG_DECOMPRESS_XZ=y
@@ -3897,8 +3897,8 @@ CONFIG_CEC_CORE=m
 CONFIG_CDROM=m
 CONFIG_CDROM_PKTCDVD=m
 CONFIG_CDROM_PKTCDVD_BUFFERS=8
-CONFIG_CC_VERSION_TEXT="gcc (Alpine 9.3.0) 9.3.0"
-CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE=y
+CONFIG_CC_VERSION_TEXT="gcc (Alpine 10.3.1) 10.3.1"
+CONFIG_CC_OPTIMIZE_FOR_SIZE=y
 CONFIG_CC_IS_GCC=y
 CONFIG_CC_HAS_WORKING_NOSANITIZE_ADDRESS=y
 CONFIG_CC_HAS_SANE_STACKPROTECTOR=y
@@ -6138,7 +6138,7 @@ CONFIG_60XX_WDT=m
 # CONFIG_CDROM_PKTCDVD_WCACHE is not set
 # CONFIG_CCS811 is not set
 # CONFIG_CC10001_ADC is not set
-# CONFIG_CC_OPTIMIZE_FOR_SIZE is not set
+# CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE is not set
 # CONFIG_CB710_DEBUG is not set
 # CONFIG_CARMINE_DRAM_CUSTOM is not set
 # CONFIG_CAN is not set

config/primary/standard.yml → config/primary/default.yml

@@ -2,9 +2,10 @@
 # SPDX-FileCopyrightText: 2021 Kaelan Thijs Fouwels <[email protected]>
 #
 # SPDX-License-Identifier: Apache-2.0
+
 header:
-  site: standard
-  comment: standard configuration
+  site: default
+  comment: default primary configuration baked into the EFI image
 primary:
   modules:
     - ahci

config/secondary/default.yml

@@ -2,9 +2,10 @@
 # SPDX-FileCopyrightText: 2021 Kaelan Thijs Fouwels <[email protected]>
 #
 # SPDX-License-Identifier: Apache-2.0
+
 header:
   site: default
-  comment: default secondary configuration installed during provisioning
+  comment: default secondary configuration copied to the filesystem during auto-provisioning
 secondary:
   modules: 
     - kvm
@@ -28,5 +29,8 @@ secondary:
     servers:
       - 0.uk.pool.ntp.org
       - 1.uk.pool.ntp.org
+
   authenticators:
-    root: JDJhJDEwJGNZQUdyMWRCb1hLcXFuL0tlMmVXbi5la3V1TlFLTkRvcWJ2SGVyUVVJRDlvMFhqckN0SC5p
+    root: 
+      mode: password
+      value: JDJhJDEwJGNZQUdyMWRCb1hLcXFuL0tlMmVXbi5la3V1TlFLTkRvcWJ2SGVyUVVJRDlvMFhqckN0SC5p

docs/img/buildkit.png.license

@@ -0,0 +1,3 @@
+// SPDX-FileCopyrightText: 2021 Kaelan Thijs Fouwels <[email protected]>
+//
+// SPDX-License-Identifier: MIT

docs/img/init.png.license

@@ -0,0 +1,4 @@
+// SPDX-FileCopyrightText: Copyright 2019 Mark Percival <[email protected]>
+// SPDX-FileCopyrightText: 2021 Kaelan Thijs Fouwels <[email protected]>
+//
+// SPDX-License-Identifier: MIT

go.mod

@@ -11,10 +11,10 @@ require (
 	github.com/google/go-tpm v0.3.2
 	github.com/google/uuid v1.2.0
 	github.com/jsimonetti/rtnetlink v0.0.0-20210614053835-9c52e516c709 // indirect
-	github.com/mdp/qrterminal v1.0.1
-	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect
+	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
 	golang.org/x/sys v0.0.0-20210611083646-a4fc73990273
 	golang.zx2c4.com/wireguard v0.0.20201118 // indirect
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20210506160403-92e472f520a5
 	gopkg.in/yaml.v2 v2.4.0
+	rsc.io/qr v0.2.0
 )

go.sum

@@ -9,7 +9,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
-github.com/cilium/ebpf v0.6.1 h1:n6ZUOkSFi6OwcMeTCFaDQx2Onx2rEikQo69315MNbdc=
 github.com/cilium/ebpf v0.6.1/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -25,7 +24,6 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -115,8 +113,6 @@ github.com/mdlayher/netlink v1.4.1 h1:I154BCU+mKlIf7BgcAJB2r7QjveNPty6uNY1g9ChVf
 github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q=
 github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00 h1:qEtkL8n1DAHpi5/AOgAckwGQUlMe4+jhL/GMt+GKIks=
 github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc=
-github.com/mdp/qrterminal v1.0.1 h1:07+fzVDlPuBlXS8tB0ktTAyf+Lp1j2+2zK3fBOL5b7c=
-github.com/mdp/qrterminal v1.0.1/go.mod h1:Z33WhxQe9B6CdW37HaVqcRKzP+kByF3q/qLxOGe12xQ=
 github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=
 github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=

init/config/config.go

@@ -7,10 +7,11 @@ package config
 
 import (
 	"fmt"
-	"log"
 	"os"
 	"path/filepath"
 
+	"os-next/init/journal"
+
 	"gopkg.in/yaml.v2"
 )
 
@@ -41,7 +42,7 @@ func LoadConfig(path string, config interface{}) (e error) {
 	y.SetStrict(true)
 	err = y.Decode(config)
 	if err != nil {
-		log.Printf("Warning: %v", err)
+		journal.Logfln("Warning: %v", err)
 	}
 
 	return nil

init/config/config_test.go

@@ -6,8 +6,9 @@
 package config_test
 
 import (
-	"os-next/init/config"
 	"testing"
+
+	"os-next/init/config"
 )
 
 func TestLoads(t *testing.T) {