From 3b8059f1fa0bd91d0f230f5e7810705419ac97f5 Mon Sep 17 00:00:00 2001 From: Matt Vander Vliet Date: Tue, 22 Mar 2022 11:39:18 -0500 Subject: [PATCH] supports separate repositories for base and ue app --- deploy/README.md | 40 +++++++--- deploy/base/routing/routing-pod-proxy.yaml | 1 + deploy/base/streaming/stream-matchmaker.yaml | 1 + deploy/base/streaming/stream-player.yaml | 1 + deploy/base/streaming/stream-runtime.yaml | 1 + deploy/base/turn/turn-api.yaml | 1 + deploy/base/turn/turn-daemonset.yaml | 1 + deploy/configure.sh | 79 +++++++++++++------- deploy/demo.yaml | 5 +- 9 files changed, 92 insertions(+), 38 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 4ca8c2f..44a82ea 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -10,10 +10,11 @@ Container Engine for Kubernetes (OKE) - [Turn Node Pool](#turn-node-pool) - [GPU Node Pool](#gpu-node-pool) - [Dependencies](#dependencies) - - [Build](#build) + - [Demo](#demo) + - [Image Build](#image-build) - [Service Layers](#service-layers) - [Pixel Streaming Build](#pixel-streaming-build) - - [Deploy](#deploy) + - [Custom Deploy](#custom-deploy) - [Telemetry](#telemetry) - [Install Prometheus Stack](#install-prometheus-stack) - [Add DCGM Exporter](#add-dcgm-exporter) @@ -191,7 +192,20 @@ aims to offer viability using the most basic/standard dependencies: --namespace metrics --create-namespace ``` -## Build +## Demo + +Prebuilt images are included with this repo, along with a demo +Pixel Streaming image. With a cluster configured per the instructions +above, you can deploy the entire runtime with the following: + +```sh +kubectl create ns demo +kubectl apply -f demo.yaml +``` + +> See [demo.yaml](./demo.yaml) for complete details + +## Image Build All of the services/constructs are contained within this repo with the exception of the Unreal project source code. See more on this [below](#pixel-streaming-build). @@ -236,7 +250,7 @@ Once repo access is obtained, the basic build process is as follows: 1. Tag and push to OCIR per [documentation](https://docs.oracle.com/en-us/iaas/Content/Registry/Tasks/registrypushingimagesusingthedockercli.htm). -## Deploy +## Custom Deploy Although we've used `helm` to install various objects in the kubernetes environment, this Pixel Streaming demo deployment is designed using plain @@ -279,18 +293,22 @@ this Pixel Streaming demo deployment is designed using plain # kubernetes namespace for pixel streaming NAMESPACE=pixel # container registry/repo path - OCIR_REPO=iad.ocir.io/mytenancy/pixeldemo + REPO=iad.ocir.io/mytenancy/pixeldemo # container registry secret (optional) - OCIR_SECRET= - # version (all services use same) - TAG_VERSION=latest + REPO_SECRET= + # tag version (all services use same) + IMAGE_TAG=latest + # unreal image container registry + UNREAL_REPO=iad.ocir.io/mytenancy/pixeldemo # name of the unreal container in OCIR UNREAL_IMAGE_NAME=my-pixelstream + # unreal container registry secret (optional) + UNREAL_REPO_SECRET= # version for the streamer image (can differ from the services) - UNREAL_IMAGE_VERSION=latest + UNREAL_IMAGE_TAG=latest # a hostname to use (nip.io ip example) INGRESS_HOST=my-pixelstream..nip.io - # optionally specify ingress path prefix (ie /game) + # optionally specify ingress path prefix (example: /game) INGRESS_PATH= # specify initial TURN service username TURN_USER=userx0000 @@ -301,7 +319,7 @@ this Pixel Streaming demo deployment is designed using plain # configure proxy prefix PROXY_PATH_PREFIX=/proxy # configure basic auth users (unreal/demo) https://doc.traefik.io/traefik/middlewares/http/basicauth/ - PROXY_AUTH_USERS='unreal:$apr1$AWc55mzG$TwDga0HZBRTBTGLHdDkUS/' + PROXY_AUTH_USERS= ``` 5. Use the [./configure.sh](./configure.sh) wrapper to generate a `kustomization` diff --git a/deploy/base/routing/routing-pod-proxy.yaml b/deploy/base/routing/routing-pod-proxy.yaml index 9fb4958..8f9f83e 100644 --- a/deploy/base/routing/routing-pod-proxy.yaml +++ b/deploy/base/routing/routing-pod-proxy.yaml @@ -40,6 +40,7 @@ spec: prometheus.io/port: "9113" sidecar.istio.io/rewriteAppHTTPProbers: "true" spec: + imagePullSecrets: [] # pod containers containers: # prometheus exporter diff --git a/deploy/base/streaming/stream-matchmaker.yaml b/deploy/base/streaming/stream-matchmaker.yaml index e2b90e9..edd9034 100644 --- a/deploy/base/streaming/stream-matchmaker.yaml +++ b/deploy/base/streaming/stream-matchmaker.yaml @@ -60,6 +60,7 @@ spec: annotations: sidecar.istio.io/rewriteAppHTTPProbers: "true" spec: + imagePullSecrets: [] # config volumes: - name: config diff --git a/deploy/base/streaming/stream-player.yaml b/deploy/base/streaming/stream-player.yaml index a6b053d..e032ac0 100644 --- a/deploy/base/streaming/stream-player.yaml +++ b/deploy/base/streaming/stream-player.yaml @@ -39,6 +39,7 @@ spec: prometheus.io/port: "9113" sidecar.istio.io/rewriteAppHTTPProbers: "true" spec: + imagePullSecrets: [] # pod containers containers: # prometheus exporter diff --git a/deploy/base/streaming/stream-runtime.yaml b/deploy/base/streaming/stream-runtime.yaml index aee1a70..d5fd840 100644 --- a/deploy/base/streaming/stream-runtime.yaml +++ b/deploy/base/streaming/stream-runtime.yaml @@ -46,6 +46,7 @@ spec: prometheus.io/port: "9000" sidecar.istio.io/rewriteAppHTTPProbers: "true" spec: + imagePullSecrets: [] serviceAccountName: stream-svc-account # allow a long time (1min) for pod to terminate in case of scale down operations on occupied pods terminationGracePeriodSeconds: 60 diff --git a/deploy/base/turn/turn-api.yaml b/deploy/base/turn/turn-api.yaml index 206f91e..35ebbee 100644 --- a/deploy/base/turn/turn-api.yaml +++ b/deploy/base/turn/turn-api.yaml @@ -42,6 +42,7 @@ spec: app.kubernetes.io/name: turn-api spec: + imagePullSecrets: [] serviceAccountName: turn-svc-account volumes: - name: shared diff --git a/deploy/base/turn/turn-daemonset.yaml b/deploy/base/turn/turn-daemonset.yaml index ed67c54..5a528da 100644 --- a/deploy/base/turn/turn-daemonset.yaml +++ b/deploy/base/turn/turn-daemonset.yaml @@ -23,6 +23,7 @@ spec: version: 0.0.1 app.kubernetes.io/name: turn spec: + imagePullSecrets: [] serviceAccountName: turn-svc-account terminationGracePeriodSeconds: 10 hostNetwork: true diff --git a/deploy/configure.sh b/deploy/configure.sh index c3f551f..f26abc6 100755 --- a/deploy/configure.sh +++ b/deploy/configure.sh @@ -6,10 +6,12 @@ # ---------------------------------------------------------------------- DIR=$(dirname $0) +BASE_DIR="$DIR/base" +OVERLAY_DIR="$DIR/overlay" + +# interpret env/properties arg DOTENV=$1 ENV_FILE="${DOTENV:-$DIR/.env}" -BASE="$DIR/base" -KOVERLAY="$DIR/overlay" # echo to stderr echoerr() { echo "$@" 1>&2; } @@ -23,8 +25,12 @@ else fi # validate -if [ -z "$OCIR_REPO" ]; then - echoerr "ERROR: Requires 'OCIR_REPO' variable ex: 'iad.ocir.io/mytenancy/my-repository'" +if [ -z "$REPO" ]; then + echoerr "ERROR: Requires 'REPO' variable ex: 'iad.ocir.io/mytenancy/my-repo'" + exit 1 +fi +if [ -z "$UNREAL_REPO" ]; then + echoerr "ERROR: Requires 'UNREAL_REPO' variable ex: 'iad.ocir.io/mytenancy/my-unreal-repo'" exit 1 fi if [ -z "$UNREAL_IMAGE_NAME" ]; then @@ -39,14 +45,14 @@ fi if [ -z "$NAMESPACE" ]; then echoerr "WARN: Recommended setting 'NAMESPACE' variable (default: pixel)" fi -if [ -z "$OCIR_SECRET" ]; then - echoerr "WARN: Using without 'OCIR_SECRET' private registry imagePullSecret ex: 'ocirsecret'" +if [ -z "$IMAGE_REPO_SECRET" ]; then + echoerr "WARN: Using without 'IMAGE_REPO_SECRET' private registry imagePullSecret ex: 'ocirsecret'" fi # Generate kustom overlay -echoerr "Generate kustomization overlay: $KOVERLAY/" -mkdir -p $KOVERLAY -cd $KOVERLAY +echoerr "Generate kustomization overlay: $OVERLAY_DIR/" +mkdir -p $OVERLAY_DIR +cd $OVERLAY_DIR # Generate patches echoerr "Generate patches..." @@ -113,16 +119,16 @@ fi fi # create registry secret patches -if [ -n "$OCIR_SECRET" ]; then +if [ -n "$REPO_SECRET" ]; then cat < patch-registry-secret.yaml - op: add - path: /spec/template/spec/imagePullSecrets + path: /spec/template/spec/imagePullSecrets/- value: - - name: "${OCIR_SECRET}" + name: "${REPO_SECRET}" EOF # inject into the patchesJson6902 - PATCH_IMAGE_PULLS_SECRETS=" + PATCH_IMAGE_PULL_SECRET=" # patch image registry secrets - path: patch-registry-secret.yaml target: @@ -163,6 +169,28 @@ EOF " fi +# create unreal registry secret patches +if [ -n "$UNREAL_REPO_SECRET" ]; then + +cat < patch-unreal-registry-secret.yaml +- op: add + path: /spec/template/spec/imagePullSecrets/- + value: + name: "${UNREAL_REPO_SECRET}" +EOF + # inject into the patchesJson6902 + PATCH_UNREAL_PULL_SECRET=" + # patch unreal image registry secret + - path: patch-unreal-registry-secret.yaml + target: + group: apps + version: v1 + kind: Deployment + name: stream +" +fi + + # Generate overlay echoerr "Generate kustomization.yaml..." cat < kustomization.yaml @@ -188,51 +216,52 @@ patchesStrategicMerge: - patch-turn-credential.yaml patchesJson6902: - ${PATCH_IMAGE_PULLS_SECRETS} + ${PATCH_IMAGE_PULL_SECRET} + ${PATCH_UNREAL_PULL_SECRET} ${PATCH_INGRESS} images: - # pixel streaming runtime + # pixel streaming application - name: pixelstreaming - newName: ${OCIR_REPO}/${UNREAL_IMAGE_NAME} - newTag: ${UNREAL_IMAGE_VERSION:-latest} + newName: ${UNREAL_REPO}/${UNREAL_IMAGE_NAME} + newTag: ${UNREAL_IMAGE_TAG:-latest} # turn image - name: turn - newName: ${OCIR_REPO}/turn + newName: ${REPO}/turn newTag: ${IMAGE_TAG:-latest} # turn aggregator/discovery - name: turn-api - newName: ${OCIR_REPO}/turn-api + newName: ${REPO}/turn-api newTag: ${IMAGE_TAG:-latest} # signal server - name: signalserver - newName: ${OCIR_REPO}/signalserver + newName: ${REPO}/signalserver newTag: ${IMAGE_TAG:-latest} # matchmaker - name: matchmaker - newName: ${OCIR_REPO}/matchmaker + newName: ${REPO}/matchmaker newTag: ${IMAGE_TAG:-latest} # player webview - name: player - newName: ${OCIR_REPO}/player + newName: ${REPO}/player newTag: ${IMAGE_TAG:-latest} # dynamic proxy svc - name: podproxy - newName: ${OCIR_REPO}/podproxy + newName: ${REPO}/podproxy newTag: ${IMAGE_TAG:-latest} # operator tools (kubectl, docker, jq) - name: kubetools - newName: ${OCIR_REPO}/kubetools + newName: ${REPO}/kubetools newTag: ${IMAGE_TAG:-latest} EOF -echoerr "Exec 'kubectl kustomize' from $KOVERLAY/kustomization.yaml" +echoerr "Exec 'kubectl kustomize' from $OVERLAY_DIR/kustomization.yaml" echoerr "---" kubectl kustomize . \ No newline at end of file diff --git a/deploy/demo.yaml b/deploy/demo.yaml index 827049f..b98bdc0 100644 --- a/deploy/demo.yaml +++ b/deploy/demo.yaml @@ -834,7 +834,7 @@ spec: name: rest resources: requests: - cpu: 2500m + cpu: 3750m - args: - -ec - | @@ -1321,6 +1321,7 @@ metadata: cert-manager.io/issuer: letsencrypt-prod kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/affinity: cookie + nginx.ingress.kubernetes.io/rewrite-target: /$1 nginx.ingress.kubernetes.io/session-cookie-name: pxstream-ingress labels: app.kubernetes.io/component: routing @@ -1339,5 +1340,5 @@ spec: name: router port: number: 80 - path: / + path: /(.*) pathType: Prefix