stream-runtime.yaml

# ----------------------------------------------------------------------
# Copyright (c) 2021, 2022 Oracle and/or its affiliates. All rights reserved.
# The Universal Permissive License (UPL), Version 1.0
# ----------------------------------------------------------------------

# Headless stream service used to to capture streaming endpoints for dns lookup (endpoints)
apiVersion: v1
kind: Service
metadata:
  name: stream-discovery
  labels:
    app: stream-discovery
    app.kubernetes.io/name: stream-discovery
spec:
  type: ClusterIP
  # make the service headless to get individual pod A records
  clusterIP: None
  ports:
    # advertise signaling websocket/http
    - port: 3000
      targetPort: 3000
      name: http
  selector:
    app: stream
    app.kubernetes.io/name: stream
---
# The signal server, pixel streaming, and annotation sidecar
apiVersion: apps/v1
kind: Deployment
metadata:
  name: stream
  labels:
    app.kubernetes.io/name: stream
spec:
  selector:
    matchLabels:
      app: stream
      app.kubernetes.io/name: stream
  template:
    metadata:
      labels:
        app: stream
        app.kubernetes.io/name: stream
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9000"
        sidecar.istio.io/rewriteAppHTTPProbers: "true"
    spec:
      imagePullSecrets: []
      serviceAccountName: stream-svc-account
      # allow a long time (1min) for pod to terminate in case of scale down operations on occupied pods
      terminationGracePeriodSeconds: 60
      # Require scheduling only to gpu pool nodes by label
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  # be sure to add this label to nodes within the node pool 'app.pixel/gpu=true'
                  - key: app.pixel/gpu
                    operator: Exists
      # Tolerate automatic gpu taint on the gpu pool rather than request specific
      # GPU resource limit
      tolerations:
        - key: "nvidia.com/gpu"
          effect: "NoSchedule"
          operator: "Exists"
      # app config
      volumes:
        - name: config
          configMap:
            name: pixel-app-configs
      # pod containers
      containers:
        # cirrus signal server sidecar
        - name: signaler
          image: signalserver
          imagePullPolicy: Always
          ports:
            - name: http
              containerPort: 3000
            - name: stream
              containerPort: 8888
            - name: metrics
              containerPort: 9000
          volumeMounts:
            - name: config
              mountPath: /config
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
          # args on cirrus.js entrypoint
          args:
            - --configFile=/config/cirrus.json
            - --httpPort=3000
            - --streamerPort=8888
            - --UseMatchmaker=true
            - --MatchmakerAddress=$(MATCHMAKER_SVC)
            - --MatchmakerPort=9999
            # future proofing camel casing
            - --useMatchmaker=true
            - --matchmakerAddress=$(MATCHMAKER_SVC)
            - --matchmakerPort=9999
            # # extensions
            - --ueRestEndpoint=http://localhost:8111
            - --rtcConfigSvc=http://$(TURN_API_SVC):3000
            - --metricsPort=9000
          # add probes
          livenessProbe:
            httpGet:
              path: /healthz
              port: http
          readinessProbe:
            httpGet:
              path: /healthz
              port: http
        # unreal engine pixel streaming runtime with VaRest support
        - name: streamer
          image: pixelstreaming
          imagePullPolicy: IfNotPresent
          ports:
            - name: rest
              containerPort: 8111
          resources:
            # NOTE: since GPU affinity is established above, it is not necessary
            # to specify the nvidia.com/gpu resource. Rather a CPU request that
            # corresponds to a portion of a single GPU VM is set - thereby allowing
            # overcommit on the GPU. This approach may have unpredicatable performance.
            requests:
              cpu: 3750m
            # limits:
            #   nvidia.com/gpu: 1

          # apply env configurations from configmap
          envFrom:
            - configMapRef:
                name: pixel-stream-configs
          env:
            # configure ws connection to the streamer (cirrus in sidecar)
            - name: SIGNAL_URL
              value: ws://localhost:8888
            # configure any extra args
            # - name: EXTRA_ARGS
            #   value: "-RestApiPort=8111"
        
        # annotation agent to apply pod-deletion-cost such that pod autoscaling
        # scale down prioritizes idle streamers. This approach, however, is not
        # ideal because it will generate a heap of pod updates on the apiserver
        # @see https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/#pod-deletion-cost
        - name: annotation-agent
          image: kubetools
          imagePullPolicy: Always
          command: ["/bin/sh"]
          resources:
            requests:
              cpu: 5m
          args:
            - -ec
            - |
              while true; do
                # obtain player connections from metrics
                export PLAYERS=$(curl -s "${SELF_METRICS}" | grep ^signalserver_player_socket_connections | awk '{print $2}')

                # apply player connections as the pod deletion cost
                kubectl annotate pod "${POD_NAME}" \
                  --namespace "${NAMESPACE}" \
                  --overwrite controller.kubernetes.io/pod-deletion-cost="${PLAYERS}" || true
                sleep 10
              done
          env:
            - name: SELF_METRICS
              value: http://localhost:9000/custom
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace