From 4db4faa0b86d16e9581f5c0ab686cf42b06fcd57 Mon Sep 17 00:00:00 2001 From: Jan Dobes Date: Fri, 22 Nov 2024 17:42:20 +0100 Subject: [PATCH] feat: add pg_repack cronjob RHINENG-14451 --- Dockerfile | 1 + README.md | 3 ++ deploy/clowdapp.yaml | 40 +++++++++++++++ docker-compose.yml | 16 ++++++ main.go | 3 ++ repack/repack.go | 120 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 183 insertions(+) create mode 100644 repack/repack.go diff --git a/Dockerfile b/Dockerfile index 0ee94535..19284f4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ ADD test /vuln4shift/test ADD vmsync /vuln4shift/vmsync ADD cleaner /vuln4shift/cleaner ADD expsync /vuln4shift/expsync +ADD repack /vuln4shift/repack ARG VERSION=dev diff --git a/README.md b/README.md index af70f587..abc196a6 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ Vulnerabilities detection for OpenShift images. # Sync Exploit data docker-compose run --rm vuln4shift_expsync + # Run pg_repack job + docker-compose run --rm vuln4shift_repack + # psql console docker-compose exec vuln4shift_database psql -U vuln4shift_admin vuln4shift diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index 9e73874d..979e9348 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -338,6 +338,26 @@ objects: cpu: ${CPU_REQUEST_EXPLOIT_SYNC} memory: ${MEMORY_REQUEST_EXPLOIT_SYNC} + - name: repack + schedule: ${SCHEDULE_REPACK} + suspend: ${{SUSPEND_REPACK}} + concurrencyPolicy: "Forbid" + podSpec: + image: ${IMAGE}:${IMAGE_TAG} + command: + - /engine/entrypoint.sh + - repack + env: + - name: LOGGING_LEVEL + value: ${LOGGING_LEVEL} + resources: + limits: + cpu: ${CPU_LIMIT_REPACK} + memory: ${MEMORY_LIMIT_REPACK} + requests: + cpu: ${CPU_REQUEST_REPACK} + memory: ${MEMORY_REQUEST_REPACK} + parameters: # IMAGE AND CLOWDER - name: IMAGE_TAG @@ -458,6 +478,18 @@ parameters: - name: MEMORY_REQUEST_EXPLOIT_SYNC description: Requested memory for pod value: "128Mi" +- name: CPU_LIMIT_REPACK + description: Maximum CPU limit for pod + value: 500m +- name: MEMORY_LIMIT_REPACK + description: Maximum memory limit for pod + value: "256Mi" +- name: CPU_REQUEST_REPACK + description: Requested CPU for pod + value: 200m +- name: MEMORY_REQUEST_REPACK + description: Requested memory for pod + value: "128Mi" # CRON SCHEDULES - name: SCHEDULE_VMAAS_SYNC @@ -492,6 +524,14 @@ parameters: description: Disable Cronjob required: true value: "false" +- name: SCHEDULE_REPACK + description: Cronjob schedule definition + required: true + value: 0 11 * * 1 +- name: SUSPEND_REPACK + description: Disable Cronjob + required: true + value: "false" # COMMON APP CONFIG - name: LOGGING_LEVEL diff --git a/docker-compose.yml b/docker-compose.yml index 71363feb..e16ab508 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -223,6 +223,22 @@ services: profiles: - expsync + vuln4shift_repack: + command: /vuln4shift/main repack + container_name: vuln4shift-repack + build: + context: . + dockerfile: ./Dockerfile + image: vuln4shift/app:latest + env_file: + - ./conf/common.env + - ./conf/dbadmin.env + depends_on: + vuln4shift_database: + condition: service_started + profiles: + - repack + volumes: vuln4shift-db-data: zookeeper_data: diff --git a/main.go b/main.go index 50287915..1d40266a 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "app/expsync" "app/manager" "app/pyxis" + "app/repack" "app/vmsync" "log" "os" @@ -29,6 +30,8 @@ func main() { cleaner.Start() case "expsync": expsync.Start() + case "repack": + repack.Start() default: log.Fatalf("Unknown service name: %s\n", os.Args[1]) } diff --git a/repack/repack.go b/repack/repack.go new file mode 100644 index 00000000..fb8e2953 --- /dev/null +++ b/repack/repack.go @@ -0,0 +1,120 @@ +package repack + +import ( + "fmt" + "os" + "os/exec" + + "github.com/sirupsen/logrus" + + "app/base/utils" +) + +var ( + logger *logrus.Logger + tables = map[string]string{ + "cluster": "account_id,uuid", + "cluster_image": "cluster_id", + "image": "manifest_schema2_digest,manifest_list_digest,docker_image_digest", + "image_cve": "cve_id", + "repository": "registry,repository", + "repository_image": "repository_id", + } + pgRepackArgs = []string{ + "--no-superuser-check", + "--no-password", + "-d", utils.Cfg.DbName, + "-h", utils.Cfg.DbHost, + "-p", fmt.Sprintf("%d", utils.Cfg.DbPort), + "-U", utils.Cfg.DbAdminUser, + } +) + +func init() { + var err error + logger, err = utils.CreateLogger(utils.Cfg.LoggingLevel) + if err != nil { + fmt.Println("Error setting up logger.") + os.Exit(1) + } + logger.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + }) +} + +// GetCmd returns command that calls pg_repack with table. +// Args are appended to the necessary pgRepackArgs. Stdout and stderr of the subprocess are redirected to host. +func getCmd(table string, args ...string) *exec.Cmd { + fullArgs := pgRepackArgs + fullArgs = append(fullArgs, "-I", table) + fullArgs = append(fullArgs, args...) + cmd := exec.Command("pg_repack", fullArgs...) + cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", utils.Cfg.DbAdminPassword)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd +} + +// Repack runs pg_repack with table. If columns are provided, cluster by these columns is executed as well. +func repack(table string, columns string) error { + var clusterCmd *exec.Cmd + if len(columns) > 0 { + clusterCmd = getCmd(table, "-o", columns) + } else { + clusterCmd = getCmd(table) + } + err := clusterCmd.Run() + if err != nil { + return err + } + return nil +} + +func createPgRepackExtension() error { + logger.Info("Checking pg_repack extension in DB") + conn, err := utils.GetStandardDbConnection(true) + if err != nil { + return err + } + defer conn.Close() + + var defaultVersion, installedVersion string + query := "SELECT COALESCE(default_version, ''), COALESCE(installed_version, '') FROM pg_available_extensions WHERE name = 'pg_repack'" + if err := conn.QueryRow(query).Scan(&defaultVersion, &installedVersion); err != nil { + logger.Error("Unable to get available pg_repack extension version, it may not be available") + return err + } + + if installedVersion != defaultVersion { + if installedVersion != "" { + logger.Infof("Dropping existing pg_repack extension version %s", installedVersion) + if _, err := conn.Exec("DROP EXTENSION pg_repack"); err != nil { + logger.Error("Dropping extension failed") + return err + } + } + logger.Infof("Creating pg_repack extension version %s", defaultVersion) + if _, err := conn.Exec("CREATE EXTENSION pg_repack"); err != nil { + logger.Error("Creating extension failed") + return err + } + } else { + logger.Infof("pg_repack extension version %s already exists", installedVersion) + } + return nil +} + +func Start() { + logger.Info("Starting pg_repack job.") + err := createPgRepackExtension() + if err != nil { + logger.Fatalf("Failed to create pg_repack extension, error: %s", err.Error()) + } + for table, columns := range tables { + err := repack(table, columns) + if err != nil { + logger.Fatalf("Failed to repack table %s, error: %s", table, err.Error()) + } + logger.Infof("Successfully repacked table %s", table) + } +}