Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Engine Operator deployment #49

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Engine Operator deployment #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
451c1c1
DX-97506: Automation of the engine experience in jenkins
teabot Nov 8, 2024
8320dc1
Update based on integration
teabot Dec 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 106 additions & 0 deletions charts/dremio_v2/config/engine/engine-options.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
#
# Copyright (C) 2017-2019 Dremio Corporation. This file is confidential and private property.
#

# This file uses kubernetes pod nomenclature: pods, cpu, memory. Values conform to the Kubernetes
# Quantity type:
# https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes
engineOptions:
#
# SIZES
# The number of pods to create for each engine replica, and the total memory to allocate to each
# executor pod in that replica. Note that the memory allocated to each executor container, may
# be lower depending on the podReservations setting.
sizes:
- name: 2XSmall
pods: 1
memory: 64Gi
- name: XSmall
pods: 1
memory: 128Gi
- name: Small
pods: 2
memory: 128Gi
#
# TARGET CPU CAPACITIES
# The total number to CPUs to allocate to each executor pod. Note that the number of CPUs
# allocated to the executor container may be lower depending on the resourceAllocationOffsets setting.
targetCpuCapacities:
- name: 16C
cpu: 16
- name: 32C
cpu: 32
#
# RESOURCE ALLOCATION OFFSETS
# The amount of CPU/Memory to reserve on each pod for containers other than that of the executor.
# Use action "reserve" to reserve pod resources for other containers (substractive), or "over-commit"
# to over-commit pod resources to the executor container (additive). Keys are arbitrary, used only to
# identify a default offset value.
resourceAllocationOffsets:
offsets:
- name: reserve-0-0
cpu: 0
memory: 0Gi
action: reserve
- name: reserve-2-4
cpu: 2
memory: 4Gi
action: reserve
- name: reserve-2-8
cpu: 2
memory: 8Gi
action: reserve
- name: reserve-2-16
cpu: 2
memory: 16Gi
action: reserve
defaultOffset: reserve-2-4
#
# EXECUTOR POD STORAGE OPTIONS
storage:
spillStorageSizes:
- name: 100GB
storage: 100Gi
- name: 250GB
storage: 250Gi
- name: 500GB
storage: 500Gi
defaultSpillStorageSize: 100GB
c3StorageSizes:
- name: 100GB
storage: 100Gi
- name: 250GB
storage: 250Gi
- name: 500GB
storage: 500Gi
defaultC3StorageSize: 100GB
#
# IDLE TIMEOUTS
# Periods are expressed in the ISO8601 duration format: https://en.wikipedia.org/wiki/ISO_8601#Durations.
idleTimeouts:
durations:
- PT2H
- PT1H30M
- PT1H
- PT30M
- PT15M
- PT10M
- PT5M
defaultDuration: PT2H
#
# JAVA OPTS
# Safe and curated Java options that can be applied to executor JVMs.
javaOpts:
- name: "UseGCLogFileRotation"
pattern: "-XX:+UseGCLogFileRotation"
applyByDefault: true
- name: "NumberOfGCLogFiles"
pattern: "-XX:NumberOfGCLogFiles=%s"
defaultValue: "5"
valueMatcher: "^[1-9][0-9]*$"
applyByDefault: true
- name: "GCLogFileSize"
pattern: "-XX:GCLogFileSize=%s"
defaultValue: "4000k"
valueMatcher: "^[1-9][0-9]*[kKmMgG]$"
applyByDefault: true
151 changes: 151 additions & 0 deletions charts/dremio_v2/config/engine/executor-statefulset-template.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
#
# Copyright (C) 2017-2019 Dremio Corporation. This file is confidential and private property.
#
#
# NOTE: This is NOT a concrete StatefulSet. It is a ConfigMap that contains a base StatefulSet that is used as a
# template for replica executors.
#
# Based on: https://github.com/dremio/dremio-cloud-tools/blob/master/charts/dremio_v2/templates/dremio-executor.yaml
#
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: template-dremio-executor
spec:
serviceName: "dremio-cluster-pod"
replicas: 0
podManagementPolicy: "Parallel"
revisionHistoryLimit: 1
selector:
matchLabels:
app: dremio-executor
template:
metadata:
labels:
role: dremio-cluster-pod
diagnostics-collector-role: dremio-executor
annotations:
dremio-configmap/checksum: 0
spec:
terminationGracePeriodSeconds: 720
securityContext:
fsGroup: 999
fsGroupChangePolicy: OnRootMismatch
containers:
- name: dremio-executor
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: false
runAsGroup: 999
runAsNonRoot: true
runAsUser: 999
seccompProfile:
type: RuntimeDefault
image: dremio/dremio-oss:latqest
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 0
memory: "0Gi"
volumeMounts:
- name: dremio-config
mountPath: /opt/dremio/conf
- name: dremio-hive2-config
mountPath: /opt/dremio/plugins/connectors/hive2.d
- name: dremio-hive2-config
mountPath: /opt/dremio/plugins/connectors/hive2-ee.d
- name: dremio-hive3-config
mountPath: /opt/dremio/plugins/connectors/hive3.d
- name: dremio-hive3-config
mountPath: /opt/dremio/plugins/connectors/hive3-ee.d
- name: dremio-default-executor-volume
mountPath: /opt/dremio/data
- name: dremio-default-executor-c3-0
mountPath: /opt/dremio/cloudcache/c0
env:
- name: DREMIO_MAX_HEAP_MEMORY_SIZE_MB
value: "0"
- name: DREMIO_MAX_DIRECT_MEMORY_SIZE_MB
value: "0"
- name: DREMIO_JAVA_SERVER_EXTRA_OPTS
value: >-
-XX:+UseG1GC
-XX:+AlwaysPreTouch
-Xms8g
-Xmx8g
-XX:HeapDumpPath=/opt/dremio/data
-XX:ErrorFile=/opt/dremio/data/hs_err_pid%p.log
-XX:MaxGCPauseMillis=500
-XX:InitiatingHeapOccupancyPercent=25
-XX:G1HeapRegionSize=32M
-XX:+PrintGCDetails
-XX:+PrintGCTimeStamps
-XX:+PrintGCDateStamps
-XX:+PrintAdaptiveSizePolicy
-XX:+PrintClassHistogramBeforeFullGC
-XX:+PrintClassHistogramAfterFullGC
-XX:+PrintReferenceGC
-Dzookeeper=zk-hs:2181
-Dservices.coordinator.enabled=false
-Dservices.coordinator.master.enabled=false
-Dservices.coordinator.master.embedded-zookeeper.enabled=false
-Dservices.executor.enabled=true
-Dservices.conduit.port=45679
- name: AWS_CREDENTIAL_PROFILES_FILE
value: "/opt/dremio/aws/credentials"
- name: AWS_SHARED_CREDENTIALS_FILE
value: "/opt/dremio/aws/credentials"
- name: DREMIO_LOG_TO_CONSOLE
value: "1"
command: [ "/opt/dremio/bin/dremio" ]
args: [ "start-fg" ]
ports:
- containerPort: 45678
name: server-fabric
- containerPort: 45679
name: server-conduit
initContainers:
- name: wait-for-zookeeper
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: false
runAsGroup: 999
runAsNonRoot: true
runAsUser: 999
seccompProfile:
type: RuntimeDefault
image: busybox
command: [ "sh", "-c", "until nc zk-hs 2181 -w1 > /dev/null; do echo Waiting for Zookeeper to be ready.; sleep 2; done;" ]
volumes:
- name: dremio-config
configMap:
name: dremio-config
- name: dremio-hive2-config
configMap:
name: dremio-hive2-config
- name: dremio-hive3-config
configMap:
name: dremio-hive3-config
volumeClaimTemplates:
- metadata:
name: dremio-default-executor-volume
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 128Gi
- metadata:
name: dremio-default-executor-c3-0
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 100Gi
Loading