From bcb8093ae91df88011b60592b863fcfd651af930 Mon Sep 17 00:00:00 2001 From: Mikhail Scherba Date: Fri, 20 Dec 2024 16:24:55 +0300 Subject: [PATCH] chrooted bash executor Signed-off-by: Mikhail Scherba --- go.mod | 10 +- go.sum | 32 +-- pkg/addon-operator/admission_http_server.go | 7 +- pkg/addon-operator/bootstrap.go | 1 + pkg/addon-operator/operator.go | 16 +- pkg/app/app.go | 6 + .../environment_manager/evironment_manager.go | 217 ++++++++++++++++++ pkg/module_manager/models/hooks/dependency.go | 2 + .../models/hooks/kind/batch_hook.go | 7 +- .../models/hooks/kind/shellhook.go | 16 +- pkg/module_manager/models/modules/basic.go | 78 +++++-- pkg/module_manager/models/modules/global.go | 4 +- pkg/module_manager/module_manager.go | 35 ++- .../extenders/script_enabled/script.go | 7 +- .../scheduler/extenders/static/static.go | 2 +- pkg/utils/chroot.go | 15 ++ 16 files changed, 382 insertions(+), 73 deletions(-) create mode 100644 pkg/module_manager/environment_manager/evironment_manager.go create mode 100644 pkg/utils/chroot.go diff --git a/go.mod b/go.mod index 4f4030aa..09ebfd02 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/dominikbraun/graph v0.23.0 github.com/ettle/strcase v0.2.0 github.com/flant/kube-client v1.2.2 - github.com/flant/shell-operator v0.0.0-20241209162655-7e40c61f7666 - github.com/go-chi/chi/v5 v5.1.0 + github.com/flant/shell-operator v0.0.0-20250205071823-e93862dadff5 + github.com/go-chi/chi/v5 v5.2.0 github.com/go-openapi/loads v0.19.5 github.com/go-openapi/spec v0.19.8 github.com/go-openapi/strfmt v0.19.5 @@ -20,7 +20,7 @@ require ( github.com/gofrs/uuid/v5 v5.3.0 github.com/hashicorp/go-multierror v1.1.1 github.com/kennygrant/sanitize v1.2.4 - github.com/onsi/gomega v1.35.1 + github.com/onsi/gomega v1.36.2 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.10.0 github.com/tidwall/gjson v1.14.4 @@ -163,10 +163,10 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.8.0 // indirect + golang.org/x/time v0.9.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.35.1 // indirect + google.golang.org/protobuf v1.36.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 4286c289..61e85ada 100644 --- a/go.sum +++ b/go.sum @@ -138,8 +138,8 @@ github.com/flant/kube-client v1.2.2 h1:27LBs+PKJEFnkQXjPU9eIps7a7iyI13AKcSYj897D github.com/flant/kube-client v1.2.2/go.mod h1:eMa3aJ6V1PRWSQ/RCROkObDpY4S74uM84SJS4G/LINg= github.com/flant/libjq-go v1.6.3-0.20201126171326-c46a40ff22ee h1:evii83J+/6QGNvyf6tjQ/p27DPY9iftxIBb37ALJRTg= github.com/flant/libjq-go v1.6.3-0.20201126171326-c46a40ff22ee/go.mod h1:f+REaGl/+pZR97rbTcwHEka/MAipoQQ2Mc0iQUj4ak0= -github.com/flant/shell-operator v0.0.0-20241209162655-7e40c61f7666 h1:Bkm4Aj46tOyEjz4+Oa6Ez1XNzhfC/2ywqJRNBYIxwxQ= -github.com/flant/shell-operator v0.0.0-20241209162655-7e40c61f7666/go.mod h1:wiD1nV16pmmAXzE5yZLM9QiAJEMjwlTR3XZSuKbpJXU= +github.com/flant/shell-operator v0.0.0-20250205071823-e93862dadff5 h1:QlxCuAKO/M8bjrDcUjE7bjgkTrs8ZE73rWIpdYcoUFg= +github.com/flant/shell-operator v0.0.0-20250205071823-e93862dadff5/go.mod h1:pyR9mte3tgcocQJPgyTH2wTzm6JsQQOuRElrd92O2Ks= github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= @@ -149,8 +149,8 @@ github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUork github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= -github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.2.0 h1:Aj1EtB0qR2Rdo2dG4O94RIU35w2lvQSj6BRA4+qwFL0= +github.com/go-chi/chi/v5 v5.2.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= @@ -283,8 +283,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= -github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -437,10 +437,10 @@ github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/R github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo= -github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/onsi/ginkgo/v2 v2.22.1 h1:QW7tbJAUDyVDVOM5dFa7qaybo+CRfR7bemlQUN6Z8aM= +github.com/onsi/ginkgo/v2 v2.22.1/go.mod h1:S6aTpoRsSq2cZOd+pssHAlKW/Q/jZt6cPrPlnj4a1xM= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -701,8 +701,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= -golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -719,8 +719,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -750,8 +750,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/addon-operator/admission_http_server.go b/pkg/addon-operator/admission_http_server.go index 3a6a2e79..25a51620 100644 --- a/pkg/addon-operator/admission_http_server.go +++ b/pkg/addon-operator/admission_http_server.go @@ -2,6 +2,7 @@ package addon_operator import ( "context" + "errors" "fmt" "log/slog" "net/http" @@ -57,7 +58,11 @@ func (as *AdmissionServer) start(ctx context.Context) { cert := path.Join(as.certsDir, "tls.crt") key := path.Join(as.certsDir, "tls.key") if err := srv.ListenAndServeTLS(cert, key); err != nil { - log.Fatal("admission server listen and serve tls", log.Err(err)) + if errors.Is(err, http.ErrServerClosed) { + log.Info("admission server stopped") + } else { + log.Fatal("admission server listen and serve tls", log.Err(err)) + } } }() diff --git a/pkg/addon-operator/bootstrap.go b/pkg/addon-operator/bootstrap.go index e31df581..77b8d52f 100644 --- a/pkg/addon-operator/bootstrap.go +++ b/pkg/addon-operator/bootstrap.go @@ -84,6 +84,7 @@ func (op *AddonOperator) SetupModuleManager(modulesDir string, globalHooksDir st ModulesDir: modulesDir, GlobalHooksDir: globalHooksDir, TempDir: tempDir, + ChrootDir: app.ShellChrootDir, } deps := module_manager.ModuleManagerDependencies{ KubeObjectPatcher: op.engine.ObjectPatcher, diff --git a/pkg/addon-operator/operator.go b/pkg/addon-operator/operator.go index 6ca4521d..593fc01a 100644 --- a/pkg/addon-operator/operator.go +++ b/pkg/addon-operator/operator.go @@ -832,14 +832,16 @@ func (op *AddonOperator) HandleConvergeModules(t sh_task.Task, logLabels map[str enabledModules[enabledModule] = struct{}{} } - for _, moduleName := range op.ModuleManager.GetModuleNames() { - if _, enabled := enabledModules[moduleName]; !enabled { - op.ModuleManager.SendModuleEvent(events.ModuleEvent{ - ModuleName: moduleName, - EventType: events.ModuleDisabled, - }) + go func() { + for _, moduleName := range op.ModuleManager.GetModuleNames() { + if _, enabled := enabledModules[moduleName]; !enabled { + op.ModuleManager.SendModuleEvent(events.ModuleEvent{ + ModuleName: moduleName, + EventType: events.ModuleDisabled, + }) + } } - } + }() } tasks := op.CreateConvergeModulesTasks(state, t.GetLogLabels(), string(taskEvent)) diff --git a/pkg/app/app.go b/pkg/app/app.go index 17c49e6b..a8a4295b 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -33,6 +33,7 @@ var ( GlobalHooksDir = "global-hooks" ModulesDir = "modules" + ShellChrootDir = "" UnnumberedModuleOrder = 1 @@ -166,6 +167,11 @@ func DefineStartCommandFlags(kpApp *kingpin.Application, cmd *kingpin.CmdClause) Default(CRDsFilters). StringVar(&CRDsFilters) + cmd.Flag("shell-chroot-dir", "Defines the path where shell scripts (shell hooks and enabled scripts) will be chrooted to."). + Envar("ADDON_OPERATOR_SHELL_CHROOT_DIR"). + Default(""). + StringVar(&ShellChrootDir) + shapp.DefineKubeClientFlags(cmd) shapp.DefineJqFlags(cmd) shapp.DefineLoggingFlags(cmd) diff --git a/pkg/module_manager/environment_manager/evironment_manager.go b/pkg/module_manager/environment_manager/evironment_manager.go new file mode 100644 index 00000000..943921cc --- /dev/null +++ b/pkg/module_manager/environment_manager/evironment_manager.go @@ -0,0 +1,217 @@ +package environment_manager + +import ( + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "sync" + "syscall" + + "github.com/deckhouse/deckhouse/pkg/log" + + "github.com/flant/addon-operator/pkg/utils" +) + +type ( + Type string + Environment int +) + +const ( + Mount Type = "mount" + File Type = "file" + DevNull Type = "devNull" +) + +const ( + NoEnvironment Environment = iota + EnabledScriptEnvironment + ShellHookEnvironment +) + +type ObjectDescriptor struct { + Source string + Target string + Flags uintptr + Type Type + TargetEnvironment Environment +} + +type Manager struct { + objects map[string]ObjectDescriptor + chroot string + + l sync.Mutex + preparedEnvironments map[string]Environment + + logger *log.Logger +} + +func NewManager(chroot string, logger *log.Logger) *Manager { + return &Manager{ + preparedEnvironments: make(map[string]Environment), + chroot: chroot, + objects: make(map[string]ObjectDescriptor), + logger: logger, + } +} + +func (m *Manager) AddObjectsToEnvironment(objects ...ObjectDescriptor) { + for _, object := range objects { + m.objects[object.Source] = object + } +} + +func makedev(majorNumber int64, minorNumber int64) int { + return int((majorNumber << 8) | (minorNumber & 0xff) | ((minorNumber & 0xfff00) << 12)) +} + +func (m *Manager) DisassembleEnvironmentForModule(moduleName, modulePath string, targetEnvironment Environment) error { + logEntry := utils.EnrichLoggerWithLabels(m.logger, map[string]string{ + "operator.component": "EnvironmentManager.DisassembleEnvironmentForModule", + }) + m.l.Lock() + defer m.l.Unlock() + + currentEnvironment := m.preparedEnvironments[moduleName] + if currentEnvironment == NoEnvironment { + return nil + } + + logEntry.Debug("Disassembling environment", + slog.String("module", moduleName), + slog.Any("currentEnvironment", currentEnvironment), + slog.Any("targetEnvironment", targetEnvironment)) + + chrootedModuleEnvPath := filepath.Join(m.chroot, moduleName) + for _, properties := range m.objects { + if targetEnvironment != currentEnvironment && properties.TargetEnvironment > targetEnvironment { + var chrootedObjectPath string + if len(properties.Target) > 0 { + chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Target) + } else { + chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Source) + } + + switch properties.Type { + case File, DevNull: + if err := os.Remove(chrootedObjectPath); err != nil { + return fmt.Errorf("delete file %q: %w", chrootedObjectPath, err) + } + + case Mount: + if err := syscall.Unmount(chrootedObjectPath, 0); err != nil { + return fmt.Errorf("unmount folder %q: %w", chrootedObjectPath, err) + } + } + } + } + + if targetEnvironment == NoEnvironment { + chrootedModuleDir := filepath.Join(chrootedModuleEnvPath, modulePath) + if err := syscall.Unmount(chrootedModuleDir, 0); err != nil { + return fmt.Errorf("unmount %q module's dir: %w", modulePath, err) + } + + delete(m.preparedEnvironments, moduleName) + } else { + m.preparedEnvironments[moduleName] = targetEnvironment + } + + return nil +} + +func (m *Manager) AssembleEnvironmentForModule(moduleName, modulePath string, targetEnvironment Environment) error { + logEntry := utils.EnrichLoggerWithLabels(m.logger, map[string]string{ + "operator.component": "EnvironmentManager.PrepareEnvironmentForModule", + }) + + m.l.Lock() + defer m.l.Unlock() + + currentEnvironment := m.preparedEnvironments[moduleName] + if currentEnvironment >= targetEnvironment { + return nil + } + + logEntry.Debug("Preparing environment", + slog.String("module", moduleName), + slog.Any("currentEnvironment", currentEnvironment), + slog.Any("targetEnvironment", targetEnvironment)) + + chrootedModuleEnvPath := filepath.Join(m.chroot, moduleName) + + if currentEnvironment == NoEnvironment { + logEntry.Debug("Preparing environment - creating the module's directory", + slog.String("module", moduleName), + slog.Any("currentEnvironment", currentEnvironment), + slog.Any("targetEnvironment", targetEnvironment)) + + chrootedModuleDir := filepath.Join(chrootedModuleEnvPath, modulePath) + if err := os.MkdirAll(chrootedModuleDir, 0o755); err != nil { + return fmt.Errorf("make %q module's dir: %w", modulePath, err) + } + + if err := syscall.Mount(modulePath, chrootedModuleDir, "", syscall.MS_BIND|syscall.MS_RDONLY, ""); err != nil { + return fmt.Errorf("mount %q module's dir: %w", modulePath, err) + } + } + + for _, properties := range m.objects { + if properties.TargetEnvironment != currentEnvironment && properties.TargetEnvironment <= targetEnvironment { + var chrootedObjectPath string + if len(properties.Target) > 0 { + chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Target) + } else { + chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Source) + } + + switch properties.Type { + case File: + if err := os.MkdirAll(filepath.Dir(chrootedObjectPath), 0o755); err != nil { + return fmt.Errorf("make dir %q: %w", chrootedObjectPath, err) + } + + bytesRead, err := os.ReadFile(properties.Source) + if err != nil { + return fmt.Errorf("read from file %q: %w", properties.Source, err) + } + + if err = os.WriteFile(chrootedObjectPath, bytesRead, 0o644); err != nil { + return fmt.Errorf("write to file %q: %w", chrootedObjectPath, err) + } + + case DevNull: + if err := os.MkdirAll(filepath.Dir(chrootedObjectPath), 0o755); err != nil { + return fmt.Errorf("make dir %q: %w", chrootedObjectPath, err) + } + + if err := syscall.Mknod(chrootedObjectPath, syscall.S_IFCHR|0o666, makedev(1, 3)); err != nil { + if errors.Is(err, os.ErrExist) { + continue + } + return fmt.Errorf("create null file: %w", err) + } + + if err := os.Chmod(chrootedObjectPath, 0o666); err != nil { + return fmt.Errorf("chmod %q file: %w", chrootedObjectPath, err) + } + + case Mount: + if err := os.MkdirAll(chrootedObjectPath, 0o755); err != nil { + return fmt.Errorf("make dir %q: %w", chrootedObjectPath, err) + } + + if err := syscall.Mount(properties.Source, chrootedObjectPath, "", properties.Flags, ""); err != nil { + return fmt.Errorf("mount folder %q: %w", chrootedObjectPath, err) + } + } + } + } + + m.preparedEnvironments[moduleName] = targetEnvironment + + return nil +} diff --git a/pkg/module_manager/models/hooks/dependency.go b/pkg/module_manager/models/hooks/dependency.go index 479b4735..5379b8d8 100644 --- a/pkg/module_manager/models/hooks/dependency.go +++ b/pkg/module_manager/models/hooks/dependency.go @@ -3,6 +3,7 @@ package hooks import ( "context" + environmentmanager "github.com/flant/addon-operator/pkg/module_manager/environment_manager" gohook "github.com/flant/addon-operator/pkg/module_manager/go_hook" "github.com/flant/addon-operator/pkg/module_manager/models/hooks/kind" "github.com/flant/addon-operator/pkg/utils" @@ -42,6 +43,7 @@ type HookExecutionDependencyContainer struct { KubeObjectPatcher kubeObjectPatcher MetricStorage metricStorage GlobalValuesGetter globalValuesGetter + EnvironmentManager *environmentmanager.Manager } type executableHook interface { diff --git a/pkg/module_manager/models/hooks/kind/batch_hook.go b/pkg/module_manager/models/hooks/kind/batch_hook.go index ad2d9d53..508134c6 100644 --- a/pkg/module_manager/models/hooks/kind/batch_hook.go +++ b/pkg/module_manager/models/hooks/kind/batch_hook.go @@ -30,6 +30,7 @@ import ( var _ gohook.HookConfigLoader = (*BatchHook)(nil) type BatchHook struct { + moduleName string sh_hook.Hook // hook ID in batch ID uint @@ -37,8 +38,9 @@ type BatchHook struct { } // NewBatchHook new hook, which runs via the OS interpreter like bash/python/etc -func NewBatchHook(name, path string, id uint, keepTemporaryHookFiles bool, logProxyHookJSON bool, logger *log.Logger) *BatchHook { +func NewBatchHook(name, path, moduleName string, id uint, keepTemporaryHookFiles bool, logProxyHookJSON bool, logger *log.Logger) *BatchHook { return &BatchHook{ + moduleName: moduleName, Hook: sh_hook.Hook{ Name: name, Path: path, @@ -143,7 +145,8 @@ func (h *BatchHook) Execute(configVersion string, bContext []bindingcontext.Bind envs). WithLogProxyHookJSON(shapp.LogProxyHookJSON). WithLogProxyHookJSONKey(h.LogProxyHookJSONKey). - WithLogger(h.Logger.Named("executor")) + WithLogger(h.Logger.Named("executor")). + WithChroot(utils.GetModuleChrootPath(h.moduleName)) usage, err := cmd.RunAndLogLines(logLabels) result.Usage = usage diff --git a/pkg/module_manager/models/hooks/kind/shellhook.go b/pkg/module_manager/models/hooks/kind/shellhook.go index 7ecc2998..737ccb1a 100644 --- a/pkg/module_manager/models/hooks/kind/shellhook.go +++ b/pkg/module_manager/models/hooks/kind/shellhook.go @@ -1,6 +1,7 @@ package kind import ( + "bytes" "fmt" "log/slog" "os" @@ -27,14 +28,16 @@ import ( var _ gohook.HookConfigLoader = (*ShellHook)(nil) type ShellHook struct { + moduleName string sh_hook.Hook ScheduleConfig *HookScheduleConfig } // NewShellHook new hook, which runs via the OS interpreter like bash/python/etc -func NewShellHook(name, path string, keepTemporaryHookFiles bool, logProxyHookJSON bool, logger *log.Logger) *ShellHook { +func NewShellHook(name, path, moduleName string, keepTemporaryHookFiles bool, logProxyHookJSON bool, logger *log.Logger) *ShellHook { return &ShellHook{ + moduleName: moduleName, Hook: sh_hook.Hook{ Name: name, Path: path, @@ -136,7 +139,8 @@ func (sh *ShellHook) Execute(configVersion string, bContext []bindingcontext.Bin envs). WithLogProxyHookJSON(shapp.LogProxyHookJSON). WithLogProxyHookJSONKey(sh.LogProxyHookJSONKey). - WithLogger(sh.Logger.Named("executor")) + WithLogger(sh.Logger.Named("executor")). + WithChroot(utils.GetModuleChrootPath(sh.moduleName)) usage, err := cmd.RunAndLogLines(logLabels) result.Usage = usage @@ -176,6 +180,7 @@ func (sh *ShellHook) getConfig() ([]byte, error) { envs := make([]string, 0) envs = append(envs, os.Environ()...) args := []string{"--config"} + var stderrBuf bytes.Buffer cmd := executor.NewExecutor( "", @@ -185,7 +190,9 @@ func (sh *ShellHook) getConfig() ([]byte, error) { WithLogProxyHookJSON(shapp.LogProxyHookJSON). WithLogProxyHookJSONKey(sh.LogProxyHookJSONKey). WithLogger(sh.Logger.Named("executor")). - WithCMDStdout(nil) + WithCMDStdout(nil). + WithCMDStderr(&stderrBuf). + WithChroot(utils.GetModuleChrootPath(sh.moduleName)) sh.Hook.Logger.Debug("Executing hook", slog.String("args", strings.Join(args, " "))) @@ -195,7 +202,8 @@ func (sh *ShellHook) getConfig() ([]byte, error) { sh.Hook.Logger.Debug("Hook config failed", slog.String("hook", sh.Name), log.Err(err), - slog.String("output", string(output))) + slog.String("output", string(output)), + slog.String("stderr", stderrBuf.String())) return nil, err } diff --git a/pkg/module_manager/models/modules/basic.go b/pkg/module_manager/models/modules/basic.go index ac84f531..70494567 100644 --- a/pkg/module_manager/models/modules/basic.go +++ b/pkg/module_manager/models/modules/basic.go @@ -21,6 +21,7 @@ import ( "github.com/flant/addon-operator/pkg/app" "github.com/flant/addon-operator/pkg/hook/types" + environmentmanager "github.com/flant/addon-operator/pkg/module_manager/environment_manager" "github.com/flant/addon-operator/pkg/module_manager/models/hooks" "github.com/flant/addon-operator/pkg/module_manager/models/hooks/kind" "github.com/flant/addon-operator/pkg/utils" @@ -192,18 +193,31 @@ func (bm *BasicModule) ResetState() { } } -// RegisterHooks find and registers all module hooks from a filesystem or GoHook Registry +// RegisterHooks searches and registers all module hooks from a filesystem or GoHook Registry func (bm *BasicModule) RegisterHooks(logger *log.Logger) ([]*hooks.ModuleHook, error) { if bm.hooks.registered { logger.Debug("Module hooks already registered") return nil, nil } - logger.Debug("Search and register hooks") - - hks, err := bm.searchAndRegisterHooks(logger) + hks, err := bm.searchModuleHooks() if err != nil { - return nil, fmt.Errorf("search and register hooks: %w", err) + return nil, fmt.Errorf("search module hooks failed: %w", err) + } + + logger.Debug("Found hooks", slog.Int("count", len(hks))) + if logger.GetLevel() == log.LevelDebug { + for _, h := range hks { + logger.Debug("ModuleHook", + slog.String("name", h.GetName()), + slog.String("path", h.GetPath())) + } + } + + logger.Debug("Register hooks") + + if err := bm.registerHooks(hks, logger); err != nil { + return nil, fmt.Errorf("register hooks: %w", err) } bm.hooks.registered = true @@ -224,6 +238,12 @@ func (bm *BasicModule) searchModuleHooks() ([]*hooks.ModuleHook, error) { return nil, fmt.Errorf("search module batch hooks: %w", err) } + if len(shellHooks)+len(batchHooks) > 0 { + if err := bm.AssembleEnvironmentForModule(environmentmanager.ShellHookEnvironment); err != nil { + return nil, fmt.Errorf("Assemble %q module's environment: %w", bm.Name, err) + } + } + mHooks := make([]*hooks.ModuleHook, 0, len(shellHooks)+len(goHooks)) for _, sh := range shellHooks { @@ -281,7 +301,7 @@ func (bm *BasicModule) searchModuleShellHooks() ([]*kind.ShellHook, error) { bm.logger.Warn("get batch hook config", slog.String("hook_file_path", hookPath), log.Err(err)) } - shHook := kind.NewShellHook(hookName, hookPath, bm.keepTemporaryHookFiles, shapp.LogProxyHookJSON, bm.logger.Named("shell-hook")) + shHook := kind.NewShellHook(hookName, hookPath, bm.Name, bm.keepTemporaryHookFiles, shapp.LogProxyHookJSON, bm.logger.Named("shell-hook")) hks = append(hks, shHook) } @@ -319,7 +339,7 @@ func (bm *BasicModule) searchModuleBatchHooks() ([]*kind.BatchHook, error) { for idx, cfg := range sdkcfgs { nestedHookName := fmt.Sprintf("%s:%s:%d", hookName, cfg.Metadata.Name, idx) - shHook := kind.NewBatchHook(nestedHookName, hookPath, uint(idx), bm.keepTemporaryHookFiles, shapp.LogProxyHookJSON, bm.logger.Named("batch-hook")) + shHook := kind.NewBatchHook(nestedHookName, hookPath, bm.Name, uint(idx), bm.keepTemporaryHookFiles, shapp.LogProxyHookJSON, bm.logger.Named("batch-hook")) hks = append(hks, shHook) } @@ -386,6 +406,7 @@ func IsFileBatchHook(path string, f os.FileInfo) error { // TODO: check binary another way args := []string{"hook", "list"} + o, err := exec.Command(path, args...).Output() if err != nil { return fmt.Errorf("exec file '%s': %w", path, err) @@ -403,21 +424,7 @@ func (bm *BasicModule) searchModuleGoHooks() []*kind.GoHook { return sdk.Registry().GetModuleHooks(bm.Name) } -func (bm *BasicModule) searchAndRegisterHooks(logger *log.Logger) ([]*hooks.ModuleHook, error) { - hks, err := bm.searchModuleHooks() - if err != nil { - return nil, fmt.Errorf("search module hooks failed: %w", err) - } - - logger.Debug("Found hooks", slog.Int("count", len(hks))) - if logger.GetLevel() == log.LevelDebug { - for _, h := range hks { - logger.Debug("ModuleHook", - slog.String("name", h.GetName()), - slog.String("path", h.GetPath())) - } - } - +func (bm *BasicModule) registerHooks(hks []*hooks.ModuleHook, logger *log.Logger) error { for _, moduleHook := range hks { hookLogEntry := logger.With("hook", moduleHook.GetName()). With("hook.type", "module") @@ -425,7 +432,7 @@ func (bm *BasicModule) searchAndRegisterHooks(logger *log.Logger) ([]*hooks.Modu // TODO: we could make multierr here and return all config errors at once err := moduleHook.InitializeHookConfig() if err != nil { - return nil, fmt.Errorf("module hook --config invalid: %w", err) + return fmt.Errorf("module hook --config invalid: %w", err) } bm.logger.Debug("module hook config print", slog.String("module_name", bm.Name), slog.String("hook_name", moduleHook.GetName()), slog.Any("config", moduleHook.GetHookConfig().V1)) @@ -452,7 +459,7 @@ func (bm *BasicModule) searchAndRegisterHooks(logger *log.Logger) ([]*hooks.Modu slog.String("bindings", moduleHook.GetConfigDescription())) } - return hks, nil + return nil } // GetPhase ... @@ -640,13 +647,18 @@ func (bm *BasicModule) RunEnabledScript(tmpDir string, precedingEnabledModules [ envs = append(envs, fmt.Sprintf("VALUES_PATH=%s", valuesPath)) envs = append(envs, fmt.Sprintf("MODULE_ENABLED_RESULT=%s", enabledResultFilePath)) + if err := bm.AssembleEnvironmentForModule(environmentmanager.EnabledScriptEnvironment); err != nil { + return false, fmt.Errorf("Assemble %q module's environment: %w", bm.Name, err) + } + cmd := executor.NewExecutor( "", enabledScriptPath, []string{}, envs). WithLogger(bm.logger.Named("executor")). - WithCMDStdout(nil) + WithCMDStdout(nil). + WithChroot(utils.GetModuleChrootPath(bm.Name)) usage, err := cmd.RunAndLogLines(logLabels) if usage != nil { @@ -1123,6 +1135,22 @@ func (bm *BasicModule) Validate() error { return nil } +func (bm *BasicModule) DisassembleEnvironmentForModule() error { + if bm.dc.EnvironmentManager != nil { + return bm.dc.EnvironmentManager.DisassembleEnvironmentForModule(bm.Name, bm.Path, environmentmanager.NoEnvironment) + } + + return nil +} + +func (bm *BasicModule) AssembleEnvironmentForModule(targetEnvironment environmentmanager.Environment) error { + if bm.dc.EnvironmentManager != nil { + return bm.dc.EnvironmentManager.AssembleEnvironmentForModule(bm.Name, bm.Path, targetEnvironment) + } + + return nil +} + func (bm *BasicModule) ValidateValues() error { return bm.valuesStorage.validateValues(bm.GetValues(false)) } diff --git a/pkg/module_manager/models/modules/global.go b/pkg/module_manager/models/modules/global.go index 17df4f37..c988c40b 100644 --- a/pkg/module_manager/models/modules/global.go +++ b/pkg/module_manager/models/modules/global.go @@ -548,7 +548,7 @@ func (gm *GlobalModule) searchGlobalShellHooks(hooksDir string) ([]*kind.ShellHo } } - globalHook := kind.NewShellHook(hookName, hookPath, gm.keepTemporaryHookFiles, false, gm.logger.Named("shell-hook")) + globalHook := kind.NewShellHook(hookName, hookPath, "global", gm.keepTemporaryHookFiles, false, gm.logger.Named("shell-hook")) hks = append(hks, globalHook) } @@ -600,7 +600,7 @@ func (gm *GlobalModule) searchGlobalBatchHooks(hooksDir string) ([]*kind.BatchHo for idx, cfg := range sdkcfgs { nestedHookName := fmt.Sprintf("%s-%s-%d", hookName, cfg.Metadata.Name, idx) - shHook := kind.NewBatchHook(nestedHookName, hookPath, uint(idx), gm.keepTemporaryHookFiles, false, gm.logger.Named("batch-hook")) + shHook := kind.NewBatchHook(nestedHookName, hookPath, "global", uint(idx), gm.keepTemporaryHookFiles, false, gm.logger.Named("batch-hook")) hks = append(hks, shHook) } diff --git a/pkg/module_manager/module_manager.go b/pkg/module_manager/module_manager.go index 4c6e4495..f25a4fae 100644 --- a/pkg/module_manager/module_manager.go +++ b/pkg/module_manager/module_manager.go @@ -18,6 +18,7 @@ import ( "github.com/flant/addon-operator/pkg/helm_resources_manager" . "github.com/flant/addon-operator/pkg/hook/types" "github.com/flant/addon-operator/pkg/kube_config_manager/config" + environmentmanager "github.com/flant/addon-operator/pkg/module_manager/environment_manager" gohook "github.com/flant/addon-operator/pkg/module_manager/go_hook" "github.com/flant/addon-operator/pkg/module_manager/loader" "github.com/flant/addon-operator/pkg/module_manager/loader/fs" @@ -71,6 +72,7 @@ type DirectoryConfig struct { ModulesDir string GlobalHooksDir string TempDir string + ChrootDir string } type KubeConfigManager interface { @@ -131,6 +133,8 @@ type ModuleManager struct { moduleScheduler *scheduler.Scheduler + environmentManager *environmentmanager.Manager + logger *log.Logger } @@ -143,7 +147,7 @@ func NewModuleManager(ctx context.Context, cfg *ModuleManagerConfig, logger *log // default loader, maybe we can register another one on startup fsLoader := fs.NewFileSystemLoader(cfg.DirectoryConfig.ModulesDir, logger.Named("file-system-loader")) - return &ModuleManager{ + mm := &ModuleManager{ ctx: cctx, cancel: cancel, @@ -165,6 +169,19 @@ func NewModuleManager(ctx context.Context, cfg *ModuleManagerConfig, logger *log logger: logger, } + + if len(cfg.DirectoryConfig.ChrootDir) > 0 { + mm.environmentManager = environmentmanager.NewManager(cfg.DirectoryConfig.ChrootDir, mm.logger) + } + + return mm +} + +// SetRequiredObjects sets the list of objects to implement in the chroot directory +func (mm *ModuleManager) SetRequiredObjects(objects ...environmentmanager.ObjectDescriptor) { + if mm.EnvironmentManagerEnabled() { + mm.environmentManager.AddObjectsToEnvironment(objects...) + } } func (mm *ModuleManager) Stop() { @@ -639,7 +656,7 @@ func (mm *ModuleManager) DeleteModule(moduleName string, logLabels map[string]st // Unregister module hooks. ml.DeregisterHooks() - return nil + return ml.DisassembleEnvironmentForModule() } // RunModule runs beforeHelm hook, helm upgrade --install and afterHelm or afterDeleteHelm hook @@ -1267,13 +1284,13 @@ func (mm *ModuleManager) registerModules(scriptEnabledExtender *script_extender. set := &moduleset.ModulesSet{} - // load and registry global hooks dep := &hooks.HookExecutionDependencyContainer{ HookMetricsStorage: mm.dependencies.HookMetricStorage, KubeConfigManager: mm.dependencies.KubeConfigManager, KubeObjectPatcher: mm.dependencies.KubeObjectPatcher, MetricStorage: mm.dependencies.MetricStorage, GlobalValuesGetter: mm.global, + EnvironmentManager: mm.environmentManager, } for _, mod := range mods { @@ -1285,12 +1302,12 @@ func (mm *ModuleManager) registerModules(scriptEnabledExtender *script_extender. } mod.WithDependencies(dep) - set.Add(mod) - err := mm.moduleScheduler.AddModuleVertex(mod) - if err != nil { - return err + + if err := mm.moduleScheduler.AddModuleVertex(mod); err != nil { + return fmt.Errorf("add module vertex: %w", err) } + scriptEnabledExtender.AddBasicModule(mod) mm.SendModuleEvent(events.ModuleEvent{ @@ -1332,6 +1349,10 @@ func (mm *ModuleManager) ModuleHasCRDs(moduleName string) bool { return mm.GetModule(moduleName).CRDExist() } +func (mm *ModuleManager) EnvironmentManagerEnabled() bool { + return mm.environmentManager != nil +} + // queueHasPendingModuleRunTaskWithStartup returns true if queue has pending tasks // with the type "ModuleRun" related to the module "moduleName" and DoModuleStartup is set to true. func queueHasPendingModuleRunTaskWithStartup(q *queue.TaskQueue, moduleName string) bool { diff --git a/pkg/module_manager/scheduler/extenders/script_enabled/script.go b/pkg/module_manager/scheduler/extenders/script_enabled/script.go index 19ec5ac5..0fe81179 100644 --- a/pkg/module_manager/scheduler/extenders/script_enabled/script.go +++ b/pkg/module_manager/scheduler/extenders/script_enabled/script.go @@ -17,6 +17,8 @@ import ( utils_file "github.com/flant/shell-operator/pkg/utils/file" ) +type scriptState string + const ( Name extenders.ExtenderName = "ScriptEnabled" @@ -25,8 +27,6 @@ const ( statError scriptState = "StatError" ) -type scriptState string - type Extender struct { tmpDir string basicModuleDescriptors map[string]moduleDescriptor @@ -46,7 +46,7 @@ func NewExtender(tmpDir string) (*Extender, error) { } if !info.IsDir() { - return nil, fmt.Errorf("%s path isn't a directory", tmpDir) + return nil, fmt.Errorf("%q path isn't a directory", tmpDir) } e := &Extender{ @@ -81,6 +81,7 @@ func (e *Extender) AddBasicModule(module node.ModuleInterface) { slog.String("module", module.GetName())) } } + e.basicModuleDescriptors[module.GetName()] = moduleD } diff --git a/pkg/module_manager/scheduler/extenders/static/static.go b/pkg/module_manager/scheduler/extenders/static/static.go index f275e490..d11dcf6d 100644 --- a/pkg/module_manager/scheduler/extenders/static/static.go +++ b/pkg/module_manager/scheduler/extenders/static/static.go @@ -29,7 +29,7 @@ func NewExtender(staticValuesFilePaths string) (*Extender, error) { valuesFile := filepath.Join(dir, "values.yaml") fileInfo, err := os.Stat(valuesFile) if err != nil { - log.Error("Couldn't stat file", + log.Warn("Couldn't stat file", slog.String("file", valuesFile)) continue } diff --git a/pkg/utils/chroot.go b/pkg/utils/chroot.go new file mode 100644 index 00000000..baf4d242 --- /dev/null +++ b/pkg/utils/chroot.go @@ -0,0 +1,15 @@ +package utils + +import ( + "fmt" + + "github.com/flant/addon-operator/pkg/app" +) + +func GetModuleChrootPath(moduleName string) string { + if len(app.ShellChrootDir) > 0 { + return fmt.Sprintf("%s/%s", app.ShellChrootDir, moduleName) + } + + return "" +}