From 0a1d38c4581078cc4b36dd573235e8432bc632a3 Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Wed, 15 Nov 2023 09:48:23 +0900 Subject: [PATCH] Improvement on upgrade mechanism --- .github/workflows/build-publish-binaries.yaml | 15 ++++++----- deepfence_agent/plugins/SecretScanner | 2 +- deepfence_agent/plugins/YaraHunter | 2 +- deepfence_agent/plugins/package-scanner | 2 +- deepfence_agent/run_discovery.sh | 16 ++++++------ deepfence_bootstrapper/Makefile | 4 +-- deepfence_bootstrapper/assets/config.ini | 8 +++--- deepfence_bootstrapper/controls/controls.go | 3 +++ .../router/openapi_client_controls.go | 4 +++ deepfence_bootstrapper/router/upgrade.go | 26 ++++++++++++++++++- deepfence_bootstrapper/supervisor/process.go | 21 ++++++++++++--- deepfence_ctl/cmd/graph.go | 2 +- deepfence_ctl/go.sum | 10 +++---- deepfence_worker/cronjobs/agent.go | 2 +- 14 files changed, 83 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-publish-binaries.yaml b/.github/workflows/build-publish-binaries.yaml index 05a6123710..7315f5a906 100644 --- a/.github/workflows/build-publish-binaries.yaml +++ b/.github/workflows/build-publish-binaries.yaml @@ -10,6 +10,7 @@ on: env: DF_BIN_VER: ${{ inputs.ver }} + VERSION: ${{ inputs.ver }} jobs: docker: @@ -35,12 +36,14 @@ jobs: mkdir -p /tmp/binaries/$DF_BIN_VER cd /tmp/binaries/$DF_BIN_VER id=$(docker create deepfenceio/deepfence_agent_ce:latest) - docker cp $id:/usr/local/bin/syft - > syft - docker cp $id:/home/deepfence/bin/yara-hunter/YaraHunter - > YaraHunter - docker cp $id:/home/deepfence/bin/secret-scanner/SecretScanner - > SecretScanner - docker cp $id:/usr/local/discovery/deepfence-discovery - > deepfence-discovery - docker cp $id:/opt/td-agent-bit/bin/fluent-bit - > fluent-bit - docker cp $id:/usr/local/bin/compliance_check/compliance - > compliance + docker cp $id:/bin/deepfenced self + docker cp $id:/home/deepfence/bin/package-scanner package_scanner + docker cp $id:/home/deepfence/bin/yara-hunter/YaraHunter malware_scanner + docker cp $id:/home/deepfence/bin/secret-scanner/SecretScanner secret_scanner + docker cp $id:/usr/local/discovery/deepfence-discovery discovery + docker cp $id:/opt/td-agent-bit/bin/fluent-bit fluentbit + docker cp $id:/usr/local/bin/syft syft + docker cp $id:/usr/local/bin/compliance_check/compliance compliance tar zcvf binaries.tar.gz ./* docker rm -v $id diff --git a/deepfence_agent/plugins/SecretScanner b/deepfence_agent/plugins/SecretScanner index 754e88babc..89c7db1345 160000 --- a/deepfence_agent/plugins/SecretScanner +++ b/deepfence_agent/plugins/SecretScanner @@ -1 +1 @@ -Subproject commit 754e88babc8e59dbbd0eb87a28ce8c5b0295e505 +Subproject commit 89c7db13455d3ef4d92aba74cf13ccac5ef7c0bf diff --git a/deepfence_agent/plugins/YaraHunter b/deepfence_agent/plugins/YaraHunter index aa33004f46..f1cc3af723 160000 --- a/deepfence_agent/plugins/YaraHunter +++ b/deepfence_agent/plugins/YaraHunter @@ -1 +1 @@ -Subproject commit aa33004f46bc4b885415de9fd8db26c30d6ee81a +Subproject commit f1cc3af723df93af38b87c5468765f41e82750be diff --git a/deepfence_agent/plugins/package-scanner b/deepfence_agent/plugins/package-scanner index eeb5dca811..3628e71384 160000 --- a/deepfence_agent/plugins/package-scanner +++ b/deepfence_agent/plugins/package-scanner @@ -1 +1 @@ -Subproject commit eeb5dca811c63b86894974df6e43954b38d1b05f +Subproject commit 3628e713843e2261b5626da5aab815e98b93ce1a diff --git a/deepfence_agent/run_discovery.sh b/deepfence_agent/run_discovery.sh index c5a9a511ac..1b42812843 100644 --- a/deepfence_agent/run_discovery.sh +++ b/deepfence_agent/run_discovery.sh @@ -7,7 +7,7 @@ PROBE_TRACKDEPLOADS=${DF_ENABLE_TRACKDEPLOADS:-"false"} PROBE_LOG_LEVEL=${LOG_LEVEL:-info} if [[ "$DF_CLUSTER_AGENT" == "true" ]]; then - /home/deepfence/deepfence_exe \ + exec /home/deepfence/deepfence_exe \ --mode=probe \ --probe.kubernetes.role=cluster \ --probe.log.level="$PROBE_LOG_LEVEL" \ @@ -41,20 +41,20 @@ fi if [[ "$DF_KUBERNETES_ON" == "Y" ]]; then if [[ "$CONTAINER_RUNTIME" == "containerd" ]] || [[ "$CONTAINER_RUNTIME" = "crio" ]]; then - env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=false --probe.podman=false --probe.cri=true --probe.cri.endpoint="$CRI_ENDPOINT" --probe.kubernetes="true" --probe.kubernetes.role=host --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" + exec env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=false --probe.podman=false --probe.cri=true --probe.cri.endpoint="$CRI_ENDPOINT" --probe.kubernetes="true" --probe.kubernetes.role=host --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" elif [[ "$CONTAINER_RUNTIME" == "podman" ]]; then - env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=false --probe.podman=true --probe.podman.endpoint="$CRI_ENDPOINT" --probe.cri=false --probe.kubernetes="true" --probe.kubernetes.role=host --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" + exec env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=false --probe.podman=true --probe.podman.endpoint="$CRI_ENDPOINT" --probe.cri=false --probe.kubernetes="true" --probe.kubernetes.role=host --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" else - env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=true --probe.podman=false --probe.cri=false --probe.kubernetes="true" --probe.kubernetes.role=host --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" + exec env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=true --probe.podman=false --probe.cri=false --probe.kubernetes="true" --probe.kubernetes.role=host --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" fi else if [[ "$DF_SERVERLESS" == "true" ]]; then - env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=false --probe.podman=false --probe.cri=false --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.conntrack=false --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" + exec env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=false --probe.podman=false --probe.cri=false --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.conntrack=false --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" elif [[ "$CONTAINER_RUNTIME" == "podman" ]]; then - env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=false --probe.podman=true --probe.podman.endpoint="$CRI_ENDPOINT" --probe.cri=false --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.conntrack=false --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" + exec env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=false --probe.podman=true --probe.podman.endpoint="$CRI_ENDPOINT" --probe.cri=false --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.conntrack=false --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" elif [[ "$CONTAINER_RUNTIME" == "unknown" ]]; then - env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=false --probe.podman=false --probe.cri=false --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.conntrack=false --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" + exec env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=false --probe.podman=false --probe.cri=false --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.conntrack=false --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" else - env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=true --probe.podman=false --probe.cri=false --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" + exec env FILEBEAT_CERT_PATH="$DF_INSTALL_DIR/etc/filebeat/filebeat.crt" CONSOLE_SERVER="https://$mgmtConsoleUrl" SCOPE_HOSTNAME="$HOSTNAME" nice -n -20 $DF_INSTALL_DIR/usr/local/discovery/deepfence-discovery --mode=probe --probe.log.level="$PROBE_LOG_LEVEL" --probe.spy.interval=5s --probe.publish.interval=10s --probe.docker.interval=10s --probe.insecure=true --probe.docker=true --probe.podman=false --probe.cri=false --probe.token="$DEEPFENCE_KEY" --probe.processes="$PROBE_PROCESSES" --probe.endpoint.report="$PROBE_CONNECTIONS" --probe.track.deploads="$PROBE_TRACKDEPLOADS" "https://$mgmtConsoleUrl" fi fi diff --git a/deepfence_bootstrapper/Makefile b/deepfence_bootstrapper/Makefile index 4983c7864e..43098d0fc4 100644 --- a/deepfence_bootstrapper/Makefile +++ b/deepfence_bootstrapper/Makefile @@ -1,11 +1,11 @@ -VERSION=`git describe --tags` +VERSION?=`git describe --tags` all: deepfence_bootstrapper local: deepfence_bootstrapper prepare: - docker run --rm -i -v $(ROOT_MAKEFILE_DIR):/src:rw -v /tmp/go:/go:rw deepfenceio/deepfence_builder_ce:$(DF_IMG_TAG) bash -c 'cd /src/deepfence_bootstrapper && make deepfence_bootstrapper' + docker run --rm -i -e VERSION=${VERSION} -v $(ROOT_MAKEFILE_DIR):/src:rw -v /tmp/go:/go:rw deepfenceio/deepfence_builder_ce:$(DF_IMG_TAG) bash -c 'cd /src/deepfence_bootstrapper && make deepfence_bootstrapper' vendor: go.mod $(shell find ../deepfence_utils -path ../deepfence_utils/vendor -prune -o -name '*.go') go mod tidy -v diff --git a/deepfence_bootstrapper/assets/config.ini b/deepfence_bootstrapper/assets/config.ini index bb9b552fb6..aba611cc9f 100644 --- a/deepfence_bootstrapper/assets/config.ini +++ b/deepfence_bootstrapper/assets/config.ini @@ -3,7 +3,7 @@ maxcpu = 10 maxmem = 100000 [process:fluentbit] -command=/bin/bash -c "$DF_INSTALL_DIR/opt/td-agent-bit/bin/fluent-bit -c $DF_INSTALL_DIR/etc/td-agent-bit/fluentbit-agent.conf" +command=/bin/bash -c "exec $DF_INSTALL_DIR/opt/td-agent-bit/bin/fluent-bit -c $DF_INSTALL_DIR/etc/td-agent-bit/fluentbit-agent.conf" path=$DF_INSTALL_DIR/opt/td-agent-bit/bin/fluent-bit autostart=true autorestart=true @@ -15,19 +15,19 @@ autostart=true autorestart=true [process:package_scanner] -command=/bin/bash -c "rm -f $DF_INSTALL_DIR/tmp/package-scanner.sock && $DF_INSTALL_DIR/home/deepfence/bin/package-scanner -socket-path $DF_INSTALL_DIR/tmp/package-scanner.sock -mode grpc-server" +command=/bin/bash -c "rm -f $DF_INSTALL_DIR/tmp/package-scanner.sock && exec $DF_INSTALL_DIR/home/deepfence/bin/package-scanner -socket-path $DF_INSTALL_DIR/tmp/package-scanner.sock -mode grpc-server" path=$DF_INSTALL_DIR/home/deepfence/bin/package-scanner autostart=true autorestart=true [process:secret_scanner] -command=/bin/bash -c "rm -f $DF_INSTALL_DIR/tmp/secret-scanner.sock && $DF_INSTALL_DIR/home/deepfence/bin/secret-scanner/SecretScanner --config-path $DF_INSTALL_DIR/home/deepfence/bin/secret-scanner --socket-path=$DF_INSTALL_DIR/tmp/secret-scanner.sock" +command=/bin/bash -c "rm -f $DF_INSTALL_DIR/tmp/secret-scanner.sock && exec $DF_INSTALL_DIR/home/deepfence/bin/secret-scanner/SecretScanner --config-path $DF_INSTALL_DIR/home/deepfence/bin/secret-scanner --socket-path=$DF_INSTALL_DIR/tmp/secret-scanner.sock" path=$DF_INSTALL_DIR/home/deepfence/bin/secret-scanner/SecretScanner autostart=true autorestart=true [process:malware_scanner] -command=/bin/bash -c "rm -f $DF_INSTALL_DIR/tmp/yara-hunter.sock && $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/YaraHunter --config-path $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter --rules-path $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/yara-rules --socket-path=$DF_INSTALL_DIR/tmp/yara-hunter.sock --enable-updater=false" +command=/bin/bash -c "rm -f $DF_INSTALL_DIR/tmp/yara-hunter.sock && exec $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/YaraHunter --config-path $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter --rules-path $DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/yara-rules --socket-path=$DF_INSTALL_DIR/tmp/yara-hunter.sock --enable-updater=false" path=$DF_INSTALL_DIR/home/deepfence/bin/yara-hunter/YaraHunter autostart=true autorestart=true diff --git a/deepfence_bootstrapper/controls/controls.go b/deepfence_bootstrapper/controls/controls.go index 085fbab1a0..6239380c70 100644 --- a/deepfence_bootstrapper/controls/controls.go +++ b/deepfence_bootstrapper/controls/controls.go @@ -35,6 +35,7 @@ func SetClusterAgentControls(k8sClusterName string) { func(req ctl.StartAgentUpgradeRequest) error { log.Info().Msg("Start Cluster Agent Upgrade") router.SetUpgrade() + defer router.UnsetUpgrade() return StartClusterAgentUpgrade(req) }) if err != nil { @@ -110,6 +111,7 @@ func SetAgentControls() { func(req ctl.StartAgentUpgradeRequest) error { log.Info().Msg("Start Agent Upgrade") router.SetUpgrade() + defer router.UnsetUpgrade() return router.StartAgentUpgrade(req) }) if err != nil { @@ -119,6 +121,7 @@ func SetAgentControls() { func(req ctl.EnableAgentPluginRequest) error { log.Info().Msg("Start & download Agent Plugin") router.SetUpgrade() + defer router.UnsetUpgrade() err = supervisor.UpgradeProcessFromURL(req.PluginName, req.BinUrl) if err != nil { return err diff --git a/deepfence_bootstrapper/router/openapi_client_controls.go b/deepfence_bootstrapper/router/openapi_client_controls.go index 3c9660f8c7..05053d5d55 100644 --- a/deepfence_bootstrapper/router/openapi_client_controls.go +++ b/deepfence_bootstrapper/router/openapi_client_controls.go @@ -134,6 +134,10 @@ func SetUpgrade() { upgrade.Store(true) } +func UnsetUpgrade() { + upgrade.Store(false) +} + func getUpgradeWorkload() int32 { if upgrade.Load() { return MAX_AGENT_WORKLOAD diff --git a/deepfence_bootstrapper/router/upgrade.go b/deepfence_bootstrapper/router/upgrade.go index e1273dacd3..a740d2ce66 100644 --- a/deepfence_bootstrapper/router/upgrade.go +++ b/deepfence_bootstrapper/router/upgrade.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "syscall" "github.com/deepfence/ThreatMapper/deepfence_bootstrapper/supervisor" ctl "github.com/deepfence/ThreatMapper/deepfence_utils/controls" @@ -49,6 +50,9 @@ func StartAgentUpgrade(req ctl.StartAgentUpgradeRequest) error { if err != nil { return err } + if info.IsDir() { + return nil + } plugins = append(plugins, NamePath{name: filepath.Base(path), path: path}) return nil }) @@ -57,16 +61,36 @@ func StartAgentUpgrade(req ctl.StartAgentUpgradeRequest) error { return err } + restart := false for _, plugin := range plugins { err = supervisor.UpgradeProcessFromFile(plugin.name, plugin.path) if err != nil { - log.Error().Msg(err.Error()) + log.Error().Msgf("plugin: %v, path: %v, err: %v", plugin.name, plugin.path, err) + } else if plugin.name == supervisor.Self_id { + restart = true } } + if restart { + log.Info().Msgf("Restart self") + restartSelf() + } + return nil } +func restartSelf() error { + errs := supervisor.StopAllProcesses() + for i := range errs { + log.Error().Msg(errs[i].Error()) + } + argv0, err := exec.LookPath(os.Args[0]) + if err != nil { + return err + } + return syscall.Exec(argv0, os.Args, os.Environ()) +} + func downloadFile(filepath string, url string) (err error) { // Create the file diff --git a/deepfence_bootstrapper/supervisor/process.go b/deepfence_bootstrapper/supervisor/process.go index 13c02d4fc8..ef2a71a339 100644 --- a/deepfence_bootstrapper/supervisor/process.go +++ b/deepfence_bootstrapper/supervisor/process.go @@ -17,7 +17,7 @@ import ( ) const ( - self_id = "self" + Self_id = "self" log_root_env = "${DF_INSTALL_DIR}/var/log/deepfenced/" EXIT_CODE_BASH_NOT_FOUND = 127 ) @@ -288,7 +288,7 @@ func WriteTo(dst, org string) error { } func UpgradeProcessFromFile(name, path string) error { - if name == self_id { + if name == Self_id { return selfUpgradeFromFile(path) } @@ -325,7 +325,7 @@ func UpgradeProcessFromFile(name, path string) error { } func UpgradeProcessFromURL(name, url string) error { - if name == self_id { + if name == Self_id { return selfUpgradeFromUrl(url) } @@ -393,6 +393,21 @@ func StopProcess(name string) error { return process.stop() } +func StopAllProcesses() []error { + + access.RLock() + defer access.RUnlock() + + errs := []error{} + for _, process := range processes { + err := process.stop() + if err != nil { + errs = append(errs, err) + } + } + return errs +} + func LoadProcess(name, path, command, env string, autorestart bool, cgroup string) { access.Lock() defer access.Unlock() diff --git a/deepfence_ctl/cmd/graph.go b/deepfence_ctl/cmd/graph.go index 37979947f6..1b9748b954 100644 --- a/deepfence_ctl/cmd/graph.go +++ b/deepfence_ctl/cmd/graph.go @@ -89,7 +89,7 @@ var graphTopologySubCmd = &cobra.Command{ root, _ := cmd.Flags().GetString("root") - var res *deepfence_server_client.ApiDocsGraphResult + var res *deepfence_server_client.ModelGraphResult var rh *stdhttp.Response switch root { case "": diff --git a/deepfence_ctl/go.sum b/deepfence_ctl/go.sum index d44f278149..93657edb1b 100644 --- a/deepfence_ctl/go.sum +++ b/deepfence_ctl/go.sum @@ -2,6 +2,7 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= @@ -14,12 +15,11 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= -github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= +github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= @@ -29,8 +29,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/deepfence_worker/cronjobs/agent.go b/deepfence_worker/cronjobs/agent.go index f49f58a35c..ca96d2d161 100644 --- a/deepfence_worker/cronjobs/agent.go +++ b/deepfence_worker/cronjobs/agent.go @@ -131,7 +131,7 @@ func ingestAgentVersion(ctx context.Context, tags_to_url map[string]string) erro if _, err = tx.Run(` UNWIND $batch as row MERGE (n:AgentVersion{node_id: row.tag}) - ON CREATE SET n.url = row.url`, + SET n.url = row.url`, map[string]interface{}{"batch": tags_to_ingest}); err != nil { return err }