From 2687fa17c2dc1795dcac6c41037e1aade502e7e1 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Fri, 4 Oct 2019 10:24:40 +0300 Subject: [PATCH 1/8] feat: versioned config from shell-operator, new kubernetes subscribe - remove obsolete glog quickfix - typo in HOOK.md - OpenAPI validation for global and module hooks config --- HOOKS.md | 2 +- cmd/addon-operator/main.go | 4 - go.mod | 12 +- go.sum | 119 ++- pkg/addon-operator/operator.go | 211 ++--- pkg/addon-operator/operator_test.go | 737 ++-------------- pkg/helm/helm.go | 2 +- pkg/module_manager/global_hook.go | 298 +++++++ pkg/module_manager/global_hook_config.go | 214 +++++ pkg/module_manager/global_hook_config_test.go | 48 + pkg/module_manager/global_hook_test.go | 7 + pkg/module_manager/hook.go | 826 ++---------------- .../hook/kube_event/hooks_controller.go | 273 ++++-- .../hook/schedule/hook_controller_test.go | 227 +++++ .../hook/schedule/hooks_controller.go | 156 ++++ pkg/module_manager/hook_executor.go | 90 ++ pkg/module_manager/module.go | 14 +- pkg/module_manager/module_hook.go | 242 +++++ pkg/module_manager/module_hook_config.go | 248 ++++++ pkg/module_manager/module_hook_config_test.go | 72 ++ pkg/module_manager/module_manager.go | 52 +- pkg/module_manager/module_manager_mock.go | 153 ++++ pkg/module_manager/module_manager_test.go | 712 ++++++++------- pkg/module_manager/utils.go | 14 + pkg/utils/module_config.go | 3 +- pkg/utils/values.go | 44 +- 26 files changed, 2704 insertions(+), 2076 deletions(-) create mode 100644 pkg/module_manager/global_hook.go create mode 100644 pkg/module_manager/global_hook_config.go create mode 100644 pkg/module_manager/global_hook_config_test.go create mode 100644 pkg/module_manager/global_hook_test.go create mode 100644 pkg/module_manager/hook/schedule/hook_controller_test.go create mode 100644 pkg/module_manager/hook/schedule/hooks_controller.go create mode 100644 pkg/module_manager/hook_executor.go create mode 100644 pkg/module_manager/module_hook.go create mode 100644 pkg/module_manager/module_hook_config.go create mode 100644 pkg/module_manager/module_hook_config_test.go create mode 100644 pkg/module_manager/module_manager_mock.go create mode 100644 pkg/module_manager/utils.go diff --git a/HOOKS.md b/HOOKS.md index 6ef67eae..a6c4843e 100644 --- a/HOOKS.md +++ b/HOOKS.md @@ -144,7 +144,7 @@ Binding context is information about the event which caused the hook execution. The $BINDING_CONTEXT_PATH environment variable contains the path to a file with JSON array of structures with the following fields: -- `binding` is a string from the `name` parameter for `schecdule` or `onKubernetesEvent` or a binding type if parameter is not set and for other hooks. For example, binding context for `beforeAll` hook: +- `binding` is a string from the `name` parameter for `schedule` or `onKubernetesEvent` or a binding type if parameter is not set and for other hooks. For example, binding context for `beforeAll` hook: ```json [{"binding":"beforeAll"}] diff --git a/cmd/addon-operator/main.go b/cmd/addon-operator/main.go index 46dc821c..ddad1347 100644 --- a/cmd/addon-operator/main.go +++ b/cmd/addon-operator/main.go @@ -1,7 +1,6 @@ package main import ( - "flag" "fmt" "os" @@ -30,9 +29,6 @@ func main() { kpApp.Command("start", "Start events processing."). Default(). Action(func(c *kingpin.ParseContext) error { - // Setting flag.Parsed() for glog. - _ = flag.CommandLine.Parse([]string{}) - // Be a good parent - clean up after the child processes // in case if addon-operator is a PID 1 process. go executor.Reap() diff --git a/go.mod b/go.mod index 7460a4e6..858a0598 100644 --- a/go.mod +++ b/go.mod @@ -4,17 +4,16 @@ go 1.12 require ( github.com/evanphx/json-patch v4.5.0+incompatible - github.com/flant/shell-operator v1.0.0-beta.5.0.20190923140739-5f7d9cca9885 // branch: release-1.0 - github.com/ghodss/yaml v1.0.0 - github.com/go-yaml/yaml v2.1.0+incompatible - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/flant/shell-operator v1.0.0-beta.5.0.20191008212016-a10d5213a3fe // branch: master + github.com/go-openapi/spec v0.19.3 github.com/kennygrant/sanitize v1.2.4 github.com/otiai10/copy v1.0.1 github.com/peterbourgon/mergemap v0.0.0-20130613134717-e21c03b7a721 github.com/prometheus/client_golang v1.0.0 github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734 - github.com/stretchr/testify v1.3.0 + github.com/sirupsen/logrus v1.2.0 + github.com/stretchr/testify v1.4.0 golang.org/x/tools v0.0.0-20190627033414-4874f863e654 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/satori/go.uuid.v1 v1.2.0 @@ -23,4 +22,7 @@ require ( k8s.io/apimachinery v0.0.0-20190409092423-760d1845f48b k8s.io/client-go v0.0.0-20190411052641-7a6b4715b709 k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 + sigs.k8s.io/yaml v1.1.0 ) + +replace github.com/go-openapi/validate => github.com/flant/go-openapi-validate v0.19.4-0.20190926112101-38fbca4ac77f // branch: fix_in_body diff --git a/go.sum b/go.sum index 354850f0..307031d1 100644 --- a/go.sum +++ b/go.sum @@ -3,10 +3,18 @@ cloud.google.com/go v0.0.0-20160913182117-3b1ae45394a2/go.mod h1:aQUYkXzVsufM+Dw cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/Azure/go-autorest v11.1.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -14,27 +22,71 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/flant/shell-operator v1.0.0-beta.2 h1:r7JHQDoXl5CLT5jLxz/TWxpjMiUrahef3W6RmAp/108= -github.com/flant/shell-operator v1.0.0-beta.2/go.mod h1:pZIR8TgEy75AcMMCpgR99r+3qcbL/R41MiiDMvtqgsc= -github.com/flant/shell-operator v1.0.0-beta.3 h1:qFD6nnim64RaQJDeXp8YYj7Ti1SOTrxpNUnQuasL5Do= -github.com/flant/shell-operator v1.0.0-beta.3/go.mod h1:qUqjq76as7qJn8BBMqCYQ+sqXBOmtk56AYY6BQQvUfA= -github.com/flant/shell-operator v1.0.0-beta.5 h1:4KYhM2YmMIWlwTiFNVzhFeoo6EPpNUyd1Tx4mOgi5jA= -github.com/flant/shell-operator v1.0.0-beta.5/go.mod h1:qUqjq76as7qJn8BBMqCYQ+sqXBOmtk56AYY6BQQvUfA= -github.com/flant/shell-operator v1.0.0-beta.5.0.20190917065053-86a4b2a7a3ae h1:mtSLQERuqx7ayExSgyO07yyVF2prG2Vkg7fCRsrl1L8= -github.com/flant/shell-operator v1.0.0-beta.5.0.20190917065053-86a4b2a7a3ae/go.mod h1:qUqjq76as7qJn8BBMqCYQ+sqXBOmtk56AYY6BQQvUfA= -github.com/flant/shell-operator v1.0.0-beta.5.0.20190923140739-5f7d9cca9885 h1:V/GLcxnepIwrunJx8M8Pr1kjg9hAMJYo4YRoF1EcBdw= -github.com/flant/shell-operator v1.0.0-beta.5.0.20190923140739-5f7d9cca9885/go.mod h1:qUqjq76as7qJn8BBMqCYQ+sqXBOmtk56AYY6BQQvUfA= +github.com/flant/go-openapi-validate v0.19.4-0.20190926112101-38fbca4ac77f h1:HTLgtnIbx2CVK74aTk1D4XbOh6tOHsPShw7UPOJTQzM= +github.com/flant/go-openapi-validate v0.19.4-0.20190926112101-38fbca4ac77f/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= +github.com/flant/shell-operator v1.0.0-beta.5.0.20191008212016-a10d5213a3fe h1:M+aKrwzUMMxroqp14zUf+EEA9+AH2BiCqtkJ1Y7cGoE= +github.com/flant/shell-operator v1.0.0-beta.5.0.20191008212016-a10d5213a3fe/go.mod h1:rJObYHs157lh7yXCHfzakhEQlWLyqlppLBLq8qbHORQ= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.4 h1:1TjOzrWkj+9BrjnM1yPAICbaoC0FyfD49oVkTBrSSa0= +github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2 h1:rf5ArTHmIJxyV5Oiks+Su0mUens1+AjpkPoWr5xFRcI= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/runtime v0.19.4 h1:csnOgcgAiuGoM/Po7PEpKDoNulCcF3FGbSnbHfxgjMI= +github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.2 h1:clPGfBnJohokno0e+d7hs6Yocrzjlgz6EsQSDncCRnE= +github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.3 h1:PAH/2DylwWcIU1s0Y7k3yNmeAgWOcKrNE2Q7Ww/kCg4= +github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -60,6 +112,10 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -74,8 +130,19 @@ github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8Nz github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= @@ -89,6 +156,7 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/otiai10/copy v1.0.1 h1:gtBjD8aq4nychvRZ2CyJvFWAw0aja+VHazDdruZKGZA= github.com/otiai10/copy v1.0.1/go.mod h1:8bMCJrAqOtN/d9oyh5HR7HhLQMvcGMpGdwRDYsfOCHc= github.com/otiai10/mint v1.2.3/go.mod h1:YnfyPNhBvnY8bW4SGQHCs/aAFhkgySlMZbrF5U0bOVw= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterbourgon/mergemap v0.0.0-20130613134717-e21c03b7a721 h1:ArxMo6jAOO2KuRsepZ0hTaH4hZCi2CCW4P9PV59HHH0= github.com/peterbourgon/mergemap v0.0.0-20130613134717-e21c03b7a721/go.mod h1:jQyRpOpE/KbvPc0VKXjAqctYglwUO5W6zAcGcFfbvlo= @@ -111,6 +179,7 @@ github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVx github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0= github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734 h1:Cpx2WLIv6fuPvaJAHNhYOgYzk/8RcJXu/8+mOrxf2KM= github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734/go.mod h1:hqVOMAwu+ekffC3Tvq5N1ljnXRrFKcaSjbCmQ8JgYaI= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= @@ -119,24 +188,41 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.1.1 h1:Sq1fR+0c58RME5EoqKdjkiQAmPjmfHlZOoRI6fTUOcs= +go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181025213731-e84da0312774 h1:a4tQYYYuK9QdeO/+kEvNYyuR21S+7ve5EANok6hABhI= golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56 h1:ZpKuNIejY8P0ExLOVyKhb0WsgG8UdvHXe6TWjY7eL6k= +golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 h1:bfLnR+k0tq5Lqt6dflRLcZiz6UaXCMt3vhYJ1l4FQ80= golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1CtlaD1KilQ6iudt1aI= golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20170412232759-a6bd8cefa181/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -150,14 +236,22 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313 h1:pczuHS43Cp2ktBEEmLwScxgjWsBSzdaQiKzUyf3DTTc= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190612231717-10539ce30318/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190627033414-4874f863e654 h1:a5iTclD5417yiTAwxzAQY6HG6HeJS4MqmPCoTlbd+0I= golang.org/x/tools v0.0.0-20190627033414-4874f863e654/go.mod h1:F+l5rz3/Uc0BJWNSxc0r6FcPZ3lxfduWytgpR5peIOQ= golang.org/x/tools/gopls v0.1.0/go.mod h1:p8Q0IUu6EEeGxqmoN/g6Et3gReLCGA7PtNRdyOxcWJE= @@ -166,6 +260,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/pkg/addon-operator/operator.go b/pkg/addon-operator/operator.go index 96361146..1c352688 100644 --- a/pkg/addon-operator/operator.go +++ b/pkg/addon-operator/operator.go @@ -1,6 +1,7 @@ package addon_operator import ( + "context" "fmt" "io" "net" @@ -10,10 +11,10 @@ import ( "path" "time" + "github.com/flant/shell-operator/pkg/hook" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/romana/rlog" - schedule_hook "github.com/flant/shell-operator/pkg/hook/schedule" "github.com/flant/shell-operator/pkg/kube" "github.com/flant/shell-operator/pkg/kube_events_manager" "github.com/flant/shell-operator/pkg/metrics_storage" @@ -24,6 +25,7 @@ import ( "github.com/flant/addon-operator/pkg/kube_config_manager" "github.com/flant/addon-operator/pkg/module_manager" kube_event_hook "github.com/flant/addon-operator/pkg/module_manager/hook/kube_event" + schedule_hook "github.com/flant/addon-operator/pkg/module_manager/hook/schedule" "github.com/flant/addon-operator/pkg/task" ) @@ -40,11 +42,11 @@ var ( // and variable changes. ModuleManager module_manager.ModuleManager - ScheduleManager schedule_manager.ScheduleManager - ScheduledHooks schedule_hook.ScheduledHooksStorage + ScheduleManager schedule_manager.ScheduleManager + ScheduleHooksController schedule_hook.ScheduleHooksController KubeEventsManager kube_events_manager.KubeEventsManager - KubeEventsHooks kube_event_hook.KubeEventsHooksController + KubernetesHooksController kube_event_hook.KubernetesHooksController MetricsStorage *metrics_storage.MetricStorage @@ -64,10 +66,10 @@ var ( // Init gets all settings, initialize managers and create a working queue. // -// Settings: directories for modules, global hooks, host name, dump file, tiller namespace. +// Settings: directories for modules and global hooks, dump file path // -// Initializing necessary objects: helm/tiller, registry manager, module manager, -// kube events manager. +// Initializing necessary objects: helm client, registry manager, module manager, +// kube config manager, kube events manager, schedule manager. // // Creating an empty queue with jobs. func Init() error { @@ -100,12 +102,19 @@ func Init() error { } rlog.Infof("INIT: Temporary dir: %s", TempDir) + // init and start metrics gathering loop MetricsStorage = metrics_storage.Init() go MetricsStorage.Run() - // Initializing the empty task queue. + + // Initializing the empty task queue and queue dumper TasksQueue = task.NewTasksQueue() + // Initializing the queue dumper, which writes queue changes to the dump file. + rlog.Debugf("INIT: Tasks queue dump file: '%s'", app.TasksQueueDumpFilePath) + queueWatcher := task.NewTasksQueueDumper(app.TasksQueueDumpFilePath, TasksQueue) + TasksQueue.AddWatcher(queueWatcher) + // Initializing the connection to the k8s. err = kube.Init(kube.InitOptions{}) @@ -114,6 +123,7 @@ func Init() error { return err } + // A useful callback when addon-operator is used as library if BeforeHelmInitCb != nil { rlog.Debugf("INIT: run BeforeHelmInitCallback") @@ -140,8 +150,8 @@ func Init() error { return err } - // Initializing module manager. + // Initializing module manager. KubeConfigManager = kube_config_manager.NewKubeConfigManager() KubeConfigManager.WithNamespace(app.Namespace) KubeConfigManager.WithConfigMapName(app.ConfigMapName) @@ -162,10 +172,6 @@ func Init() error { return err } - // Initializing the queue dumper, which writes queue changes to the dump file. - rlog.Debugf("INIT: Tasks queue dump file: '%s'", app.TasksQueueDumpFilePath) - queueWatcher := task.NewTasksQueueDumper(app.TasksQueueDumpFilePath, TasksQueue) - TasksQueue.AddWatcher(queueWatcher) // Initializing the hooks schedule. ScheduleManager, err = schedule_manager.Init() @@ -174,12 +180,25 @@ func Init() error { return err } - KubeEventsManager, err = kube_events_manager.Init() - if err != nil { - rlog.Errorf("INIT: Cannot initialize kube events manager: %s", err) - return err - } - KubeEventsHooks = kube_event_hook.NewMainKubeEventsHooksController() + ScheduleHooksController = schedule_hook.NewScheduleHooksController() + ScheduleHooksController.WithModuleManager(ModuleManager) + ScheduleHooksController.WithScheduleManager(ScheduleManager) + + KubeEventsManager = kube_events_manager.NewKubeEventsManager() + KubeEventsManager.WithContext(context.Background()) + + KubernetesHooksController = kube_event_hook.NewKubernetesHooksController() + KubernetesHooksController.WithModuleManager(ModuleManager) + KubernetesHooksController.WithKubeEventsManager(KubeEventsManager) + + + //// Initialize kube events + //KubeEventsManager, err = kube_events_manager.Init() + //if err != nil { + // rlog.Errorf("INIT: Cannot initialize kube events manager: %s", err) + // return err + //} + //KubernetesHooksController = kube_event_hook.NewKubernetesHooksController() return nil } @@ -196,7 +215,14 @@ func Run() { CreateOnStartupTasks() CreateReloadAllTasks(true) - _ = KubeEventsHooks.EnableGlobalHooks(ModuleManager, KubeEventsManager) + err := KubernetesHooksController.EnableGlobalHooks() + if err != nil { + // Something wrong with global hook configs, cannot start informers. + rlog.Errorf("Start informers for global kubernetes hooks: %v", err) + return + } + // Start all created informers + KubeEventsManager.Start() TasksQueue.ChangesEnable(true) @@ -231,15 +257,15 @@ func ManagersEventsHandler() { rlog.Infof("QUEUE add ModuleRun %s", newTask.Name) } // As module list may have changed, hook schedule index must be re-created. - ScheduledHooks = UpdateScheduleHooks(ScheduledHooks) + ScheduleHooksController.UpdateScheduleHooks() case module_manager.GlobalChanged: // Global values are changed, all modules must be restarted. rlog.Infof("EVENT GlobalChanged") TasksQueue.ChangesDisable() CreateReloadAllTasks(false) TasksQueue.ChangesEnable(true) - // Re-creating schedule hook index - ScheduledHooks = UpdateScheduleHooks(ScheduledHooks) + // As module list may have changed, hook schedule index must be re-created. + ScheduleHooksController.UpdateScheduleHooks() case module_manager.AmbigousState: rlog.Infof("EVENT AmbiguousState") TasksQueue.ChangesDisable() @@ -254,58 +280,31 @@ func ManagersEventsHandler() { rlog.Infof("QUEUE push ModuleManagerRetry, push FailedModuleDelay") } case crontab := <-schedule_manager.ScheduleCh: - scheduleHooks := ScheduledHooks.GetHooksForSchedule(crontab) - for _, hook := range scheduleHooks { - var getHookErr error - - _, getHookErr = ModuleManager.GetGlobalHook(hook.Name) - if getHookErr == nil { - for _, scheduleConfig := range hook.Schedule { - bindingName := scheduleConfig.Name - if bindingName == "" { - bindingName = module_manager.ContextBindingType[module_manager.Schedule] - } - newTask := task.NewTask(task.GlobalHookRun, hook.Name). - WithBinding(module_manager.Schedule). - AppendBindingContext(module_manager.BindingContext{Binding: bindingName}). - WithAllowFailure(scheduleConfig.AllowFailure) - TasksQueue.Add(newTask) - rlog.Debugf("QUEUE add GlobalHookRun@Schedule '%s'", hook.Name) - } - continue - } + rlog.Infof("EVENT Schedule event '%s'", crontab) - _, getHookErr = ModuleManager.GetModuleHook(hook.Name) - if getHookErr == nil { - for _, scheduleConfig := range hook.Schedule { - bindingName := scheduleConfig.Name - if bindingName == "" { - bindingName = module_manager.ContextBindingType[module_manager.Schedule] - } - newTask := task.NewTask(task.ModuleHookRun, hook.Name). - WithBinding(module_manager.Schedule). - AppendBindingContext(module_manager.BindingContext{Binding: bindingName}). - WithAllowFailure(scheduleConfig.AllowFailure) - TasksQueue.Add(newTask) - rlog.Debugf("QUEUE add ModuleHookRun@Schedule '%s'", hook.Name) - } - continue - } + tasks, err := ScheduleHooksController.HandleEvent(crontab) + if err != nil { + rlog.Errorf("MAIN_LOOP Schedule event '%s': %s", crontab, err) + break + } - rlog.Errorf("MAIN_LOOP hook '%s' scheduled but not found by module_manager", hook.Name) + for _, resTask := range tasks { + TasksQueue.Add(resTask) + rlog.Infof("QUEUE add %s@%s %s", resTask.GetType(), resTask.GetBinding(), resTask.GetName()) } + case kubeEvent := <-kube_events_manager.KubeEventCh: rlog.Infof("EVENT Kube event '%s'", kubeEvent.ConfigId) - res, err := KubeEventsHooks.HandleEvent(kubeEvent) + tasks, err := KubernetesHooksController.HandleEvent(kubeEvent) if err != nil { rlog.Errorf("MAIN_LOOP error handling kube event '%s': %s", kubeEvent.ConfigId, err) break } - for _, task := range res.Tasks { - TasksQueue.Add(task) - rlog.Infof("QUEUE add %s@%s %s", task.GetType(), task.GetBinding(), task.GetName()) + for _, t := range tasks { + TasksQueue.Add(t) + rlog.Infof("QUEUE add %s@%s %s", t.GetType(), t.GetBinding(), t.GetName()) } case <-ManagersEventsHandlerStopCh: rlog.Infof("EVENT Stop") @@ -343,7 +342,7 @@ func runDiscoverModulesState(discoverTask task.Task) error { if discoverTask.GetOnStartupHooks() { // error can be ignored, DiscoverModulesState should return existed modules disabledModule, _ := ModuleManager.GetModule(moduleName) - if err = ModuleManager.InitModuleHooks(disabledModule); err != nil { + if err = ModuleManager.RegisterModuleHooks(disabledModule); err != nil { return err } } @@ -363,7 +362,7 @@ func runDiscoverModulesState(discoverTask task.Task) error { for _, hookName := range afterAllHooks { newTask := task.NewTask(task.GlobalHookRun, hookName). WithBinding(module_manager.AfterAll). - AppendBindingContext(module_manager.BindingContext{Binding: module_manager.ContextBindingType[module_manager.AfterAll]}) + AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: module_manager.ContextBindingType[module_manager.AfterAll]}}) TasksQueue.Add(newTask) rlog.Debugf("QUEUE add GlobalHookRun@AfterAll '%s'", hookName) } @@ -371,12 +370,12 @@ func runDiscoverModulesState(discoverTask task.Task) error { rlog.Infof("QUEUE add all GlobalHookRun@AfterAll") } - ScheduledHooks = UpdateScheduleHooks(nil) + ScheduleHooksController.UpdateScheduleHooks() // Enable kube events hooks for newly enabled modules // FIXME convert to a task that run after AfterHelm if there is a flag in binding config to start informers after CRD installation. for _, moduleName := range modulesState.EnabledModules { - err = KubeEventsHooks.EnableModuleHooks(moduleName, ModuleManager, KubeEventsManager) + err = KubernetesHooksController.EnableModuleHooks(moduleName) if err != nil { return err } @@ -385,7 +384,7 @@ func runDiscoverModulesState(discoverTask task.Task) error { // TODO is queue should be cleaned from hook run tasks of deleted module? // Disable kube events hooks for newly disabled modules for _, moduleName := range modulesState.ModulesToDisable { - err = KubeEventsHooks.DisableModuleHooks(moduleName, ModuleManager, KubeEventsManager) + err = KubernetesHooksController.DisableModuleHooks(moduleName) if err != nil { return err } @@ -523,76 +522,6 @@ func TasksRunner() { } } -// UpdateScheduleHooks creates the new ScheduledHooks. -// Calculates the difference between the old and the new schedule, -// removes what was in the old but is missing in the new schedule. -func UpdateScheduleHooks(storage schedule_hook.ScheduledHooksStorage) schedule_hook.ScheduledHooksStorage { - if ScheduleManager == nil { - return nil - } - - oldCrontabs := map[string]bool{} - if storage != nil { - for _, crontab := range storage.GetCrontabs() { - oldCrontabs[crontab] = false - } - } - - newScheduledTasks := schedule_hook.ScheduledHooksStorage{} - - globalHooks := ModuleManager.GetGlobalHooksInOrder(module_manager.Schedule) -LOOP_GLOBAL_HOOKS: - for _, globalHookName := range globalHooks { - globalHook, _ := ModuleManager.GetGlobalHook(globalHookName) - for _, schedule := range globalHook.Config.Schedule { - _, err := ScheduleManager.Add(schedule.Crontab) - if err != nil { - rlog.Errorf("Schedule: cannot add '%s' for global hook '%s': %s", schedule.Crontab, globalHookName, err) - continue LOOP_GLOBAL_HOOKS - } - rlog.Debugf("Schedule: add '%s' for global hook '%s'", schedule.Crontab, globalHookName) - } - newScheduledTasks.AddHook(globalHook.Name, globalHook.Config.Schedule) - } - - modules := ModuleManager.GetModuleNamesInOrder() - for _, moduleName := range modules { - moduleHooks, _ := ModuleManager.GetModuleHooksInOrder(moduleName, module_manager.Schedule) - LOOP_MODULE_HOOKS: - for _, moduleHookName := range moduleHooks { - moduleHook, _ := ModuleManager.GetModuleHook(moduleHookName) - for _, schedule := range moduleHook.Config.Schedule { - _, err := ScheduleManager.Add(schedule.Crontab) - if err != nil { - rlog.Errorf("Schedule: cannot add '%s' for hook '%s': %s", schedule.Crontab, moduleHookName, err) - continue LOOP_MODULE_HOOKS - } - rlog.Debugf("Schedule: add '%s' for hook '%s'", schedule.Crontab, moduleHookName) - } - newScheduledTasks.AddHook(moduleHook.Name, moduleHook.Config.Schedule) - } - } - - if len(oldCrontabs) > 0 { - // Creates a new set of schedules. If the schedule is in oldCrontabs, then sets it to true. - newCrontabs := newScheduledTasks.GetCrontabs() - for _, crontab := range newCrontabs { - if _, has_crontab := oldCrontabs[crontab]; has_crontab { - oldCrontabs[crontab] = true - } - } - - // Goes through the old set of schedules and removes from processing schedules with false. - for crontab, _ := range oldCrontabs { - if !oldCrontabs[crontab] { - ScheduleManager.Remove(crontab) - } - } - } - - return newScheduledTasks -} - func CreateOnStartupTasks() { rlog.Infof("QUEUE add all GlobalHookRun@OnStartup") @@ -601,7 +530,7 @@ func CreateOnStartupTasks() { for _, hookName := range onStartupHooks { newTask := task.NewTask(task.GlobalHookRun, hookName). WithBinding(module_manager.OnStartup). - AppendBindingContext(module_manager.BindingContext{Binding: module_manager.ContextBindingType[module_manager.OnStartup]}) + AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: module_manager.ContextBindingType[module_manager.OnStartup]}}) TasksQueue.Add(newTask) rlog.Debugf("QUEUE add GlobalHookRun@OnStartup '%s'", hookName) } @@ -618,7 +547,7 @@ func CreateReloadAllTasks(onStartup bool) { for _, hookName := range beforeAllHooks { newTask := task.NewTask(task.GlobalHookRun, hookName). WithBinding(module_manager.BeforeAll). - AppendBindingContext(module_manager.BindingContext{Binding: module_manager.ContextBindingType[module_manager.BeforeAll]}) + AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: module_manager.ContextBindingType[module_manager.BeforeAll]}}) TasksQueue.Add(newTask) rlog.Debugf("QUEUE GlobalHookRun@BeforeAll '%s'", module_manager.BeforeAll, hookName) diff --git a/pkg/addon-operator/operator_test.go b/pkg/addon-operator/operator_test.go index fe53ddcd..6804f086 100644 --- a/pkg/addon-operator/operator_test.go +++ b/pkg/addon-operator/operator_test.go @@ -1,725 +1,98 @@ package addon_operator import ( - "flag" - "fmt" - "os" - "path" - "strconv" - "strings" "testing" - "time" - "github.com/golang/glog" - "github.com/romana/rlog" "github.com/stretchr/testify/assert" - "gopkg.in/satori/go.uuid.v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/flant/shell-operator/pkg/kube_events_manager" + hook2 "github.com/flant/shell-operator/pkg/hook" "github.com/flant/shell-operator/pkg/metrics_storage" - "github.com/flant/shell-operator/pkg/schedule_manager" - "github.com/flant/addon-operator/pkg/helm" - "github.com/flant/addon-operator/pkg/kube_config_manager" "github.com/flant/addon-operator/pkg/module_manager" "github.com/flant/addon-operator/pkg/task" ) -type KubeEventsHooksControllerMock struct{} - -func (obj *KubeEventsHooksControllerMock) EnableGlobalHooks(moduleManager module_manager.ModuleManager, eventsManager kube_events_manager.KubeEventsManager) error { - return nil -} - -func (obj *KubeEventsHooksControllerMock) EnableModuleHooks(moduleName string, moduleManager module_manager.ModuleManager, eventsManager kube_events_manager.KubeEventsManager) error { - return nil -} - -func (obj *KubeEventsHooksControllerMock) DisableModuleHooks(moduleName string, moduleManager module_manager.ModuleManager, eventsManager kube_events_manager.KubeEventsManager) error { - return nil -} - -func (obj *KubeEventsHooksControllerMock) HandleEvent(kubeEvent kube_events_manager.KubeEvent) (*struct{ Tasks []task.Task }, error) { - return nil, nil -} - -type KubeEventsManagerMock struct{} - -func (kem *KubeEventsManagerMock) Run(eventTypes []kube_events_manager.OnKubernetesEventType, kind, namespace string, labelSelector *metav1.LabelSelector, objectName, jqFilter string, debug bool) (string, error) { - return uuid.NewV4().String(), nil -} - -func (kem *KubeEventsManagerMock) Stop(configId string) error { - return nil -} - -type ModuleManagerMock struct { - BeforeHookErrorsCount int - TestModuleErrorsCount int - DeleteModuleErrorsCount int - ScheduledHookErrorsCount int -} - -var _ module_manager.ModuleManager = &ModuleManagerMock{} - -var mainTestGlobalHooksMap = map[module_manager.BindingType][]string{ - module_manager.OnStartup: { - "hook_1__31", "hook_2__32", - }, - module_manager.BeforeAll: { - "before_hook_1__51", "before_hook_2__52", - }, - module_manager.AfterAll: { - "after_hook_1__201", "after_hook_2__202", - }, -} - -var scheduledHooks = map[string]schedule_manager.ScheduleConfig{ - "scheduled_global_1": { - Crontab: "*/1 * * * *", - AllowFailure: true, - }, - "scheduled_global_2": { - Crontab: "*/1 * * * *", - AllowFailure: true, - }, - "scheduled_global_3": { - Crontab: "*/1 * * * *", - AllowFailure: true, - }, - "scheduled_module_1": { - Crontab: "*/2 * * * *", - AllowFailure: false, - }, -} - -var runOrder = []int{} - -var globalT *testing.T - -func (m *ModuleManagerMock) Init() error { - fmt.Println("Init ModuleManagerMock") - return nil -} -func (m *ModuleManagerMock) Run() { - fmt.Println("ModuleManagerMock Run") -} - -// Only for InitModuleHooks -func (m *ModuleManagerMock) GetModule(name string) (*module_manager.Module, error) { - return nil, nil -} - -func (m *ModuleManagerMock) GetModuleNamesInOrder() []string { - return []string{"test_module_1__101", "test_module_2__102"} -} - -func (m *ModuleManagerMock) DiscoverModulesState() (*module_manager.ModulesState, error) { - return &module_manager.ModulesState{ - EnabledModules: []string{"test_module_1__101", "test_module_2__102"}, - ModulesToDisable: []string{"disabled_module_1__111", "disabled_2__112", "disabled_3.14__113"}, - ReleasedUnknownModules: []string{"unknown_module_1__121", "abandoned_1__122", "forgotten_3.14__123"}, - }, nil -} - -func (m *ModuleManagerMock) GetGlobalHook(name string) (*module_manager.GlobalHook, error) { - if _, has_hook := scheduledHooks[name]; has_hook { - return &module_manager.GlobalHook{ - CommonHook: &module_manager.CommonHook{ - Name: name, - Path: "/addon-operator/hooks/global_1", - Bindings: []module_manager.BindingType{module_manager.Schedule}, - OrderByBinding: map[module_manager.BindingType]float64{}, - }, - Config: &module_manager.GlobalHookConfig{ - HookConfig: module_manager.HookConfig{ - Schedule: []schedule_manager.ScheduleConfig{ - scheduledHooks[name], - }, +// CreateOnStartupTasks fills a working queue with onStartup hooks. +// TaskRunner should run all hooks and clean a queue. +func Test_Operator_CreateOnStartupTasks_TaskRunner(t *testing.T) { + MetricsStorage = metrics_storage.Init() + var globalHook1 = &module_manager.GlobalHook{ + CommonHook: &module_manager.CommonHook{ + Name: "hook-global-1", + }, + Config: &module_manager.GlobalHookConfig{ + HookConfig: hook2.HookConfig{ + OnStartup: &hook2.OnStartupConfig{ + Order: 10, }, }, - }, nil - } else { - // Global hook run task handler requires Path field - return &module_manager.GlobalHook{ - CommonHook: &module_manager.CommonHook{ - Name: name, - Path: "/addon-operator/hooks/global_hook_1_1", - Bindings: []module_manager.BindingType{module_manager.BeforeAll}, - OrderByBinding: map[module_manager.BindingType]float64{}, - }, - Config: &module_manager.GlobalHookConfig{ - BeforeAll: 10, - }, - }, nil + }, } -} -func (m *ModuleManagerMock) GetModuleHook(name string) (*module_manager.ModuleHook, error) { - if _, has_hook := scheduledHooks[name]; has_hook { - return &module_manager.ModuleHook{ - CommonHook: &module_manager.CommonHook{ - Name: name, - Path: "/addon-operator/modules/000_test_modu", - Bindings: []module_manager.BindingType{module_manager.Schedule}, - OrderByBinding: map[module_manager.BindingType]float64{}, - }, - Module: &module_manager.Module{ - Name: "test_module", - DirectoryName: "/addon-operator/modules/000_test_modue", - Path: "/addon-operator/modules/000_test_modu", - }, - Config: &module_manager.ModuleHookConfig{ - HookConfig: module_manager.HookConfig{ - Schedule: []schedule_manager.ScheduleConfig{ - scheduledHooks[name], - }, + var globalHook2 = &module_manager.GlobalHook{ + CommonHook: &module_manager.CommonHook{ + Name: "hook-global-2", + }, + Config: &module_manager.GlobalHookConfig{ + HookConfig: hook2.HookConfig{ + OnStartup: &hook2.OnStartupConfig{ + Order: 10, }, }, - }, nil + }, } - return nil, nil -} -func (m *ModuleManagerMock) GetGlobalHooksInOrder(bindingType module_manager.BindingType) []string { - if bindingType == module_manager.Schedule { - res := []string{} - for k, _ := range scheduledHooks { - if strings.Contains(k, "global") { - res = append(res, k) - } - } - return res + var globalHooksMock = map[string]*module_manager.GlobalHook{ + "hook-global-1": globalHook1, + "hook-global-2": globalHook2, } - return mainTestGlobalHooksMap[bindingType] -} -func (m *ModuleManagerMock) GetModuleHooksInOrder(moduleName string, bindingType module_manager.BindingType) ([]string, error) { - if bindingType == module_manager.Schedule { - res := []string{} - for k, _ := range scheduledHooks { - if strings.Contains(k, "module") { - res = append(res, k) - } - } - return res, nil - } - return []string{"test_module_hook_1", "test_module_hook_2"}, nil -} - -func (m *ModuleManagerMock) DeleteModule(moduleName string) error { - addRunOrder(moduleName) - fmt.Printf("ModuleManagerMock DeleteModule '%s'\n", moduleName) - if strings.Contains(moduleName, "disabled_module_1") && m.DeleteModuleErrorsCount > 0 { - m.DeleteModuleErrorsCount-- - return fmt.Errorf("fake module delete error: helm run error") - } - return nil -} - -func (m *ModuleManagerMock) RunModule(moduleName string, onStartup bool) error { - addRunOrder(moduleName) - fmt.Printf("ModuleManagerMock RunModule '%s'\n", moduleName) - if strings.Contains(moduleName, "test_module_2") && m.TestModuleErrorsCount > 0 { - m.TestModuleErrorsCount-- - return fmt.Errorf("fake module error: /bin/bash not found") - } - return nil -} - -func (m *ModuleManagerMock) RunGlobalHook(hookName string, binding module_manager.BindingType, bindingContext []module_manager.BindingContext) error { - addRunOrder(hookName) - fmt.Printf("Run global hook name '%s' binding '%s'\n", hookName, binding) - if strings.Contains(hookName, "before_hook_1") && m.BeforeHookErrorsCount > 0 { - m.BeforeHookErrorsCount-- - return fmt.Errorf("fake module error: /bin/bash not found") - } - return nil -} - -func (m *ModuleManagerMock) RunModuleHook(hookName string, binding module_manager.BindingType, bindingContext []module_manager.BindingContext) error { - addRunOrder(hookName) - fmt.Printf("Run module hook name '%s' binding '%s'\n", hookName, binding) - if strings.Contains(hookName, "scheduled_module_1") && m.ScheduledHookErrorsCount > 0 { - m.ScheduledHookErrorsCount-- - return fmt.Errorf("fake module hook error: /bin/ash not found") - } - return nil -} - -func (m *ModuleManagerMock) InitModuleHooks(module *module_manager.Module) error { - return nil -} - -func (m *ModuleManagerMock) Retry() { - fmt.Println("ModuleManagerMock Retry") -} - -func (m *ModuleManagerMock) WithDirectories(modulesDir string, globalHooksDir string, tempDir string) module_manager.ModuleManager { - fmt.Println("WithDirectories") - return m -} - -func (m *ModuleManagerMock) WithKubeConfigManager(kubeConfigManager kube_config_manager.KubeConfigManager) module_manager.ModuleManager { - fmt.Println("WithKubeConfigManager") - return m -} - - -type MockHelmClient struct { - helm.HelmClient - DeleteReleaseErrorsCount int -} - -func (h MockHelmClient) CommandEnv() []string { - return []string{} -} - -func (h MockHelmClient) DeleteRelease(name string) error { - addRunOrder(name) - fmt.Printf("HelmClient: DeleteRelease '%s'\n", name) - if strings.Contains(name, "abandoned_2") && h.DeleteReleaseErrorsCount > 0 { - h.DeleteReleaseErrorsCount-- - return fmt.Errorf("fake helm error: helm syntax error") - } - return nil -} - -func addRunOrder(name string) { - if !strings.Contains(name, "__") { - return - } - order := strings.Split(name, "__")[1] - orderI, err := strconv.Atoi(order) - if err != nil { - globalT.Fatalf("Cannot parse number from order '%s' from name '%s'", order, name) - } - runOrder = append(runOrder, orderI) -} - -type QueueDumperTest struct { -} - -func (q *QueueDumperTest) QueueChangeCallback() { - headTask, _ := TasksQueue.Peek() - if headTask != nil { - fmt.Printf("head task now is '%s', len=%d\n", headTask.GetName(), TasksQueue.Length()) - } -} - -// Тест заполнения очереди заданиями при запуске и прогон TaskRunner -// после прогона очередь должна быть пустой -func TestMain_TaskRunner_CreateOnStartupTasks(t *testing.T) { - runOrder = []int{} + var hookRun = struct{ + hookGlobal1 bool + hookGlobal2 bool + }{} // Mock ModuleManager - ModuleManager = &ModuleManagerMock{} + ModuleManager = module_manager.NewModuleManagerMock(module_manager.ModuleManagerMockFns{ + GetGlobalHooksInOrder: func(bindingType module_manager.BindingType) []string { + res := []string{} + for k := range globalHooksMock{ + res = append(res, k) + } + return res + }, + GetGlobalHook: func(name string) (hook *module_manager.GlobalHook, e error) { + return globalHooksMock[name], nil + }, + RunGlobalHook: func(hookName string, binding module_manager.BindingType, bindingContext []module_manager.BindingContext) error { + switch hookName { + case "hook-global-1": + hookRun.hookGlobal1 = true + case "hook-global-2": + hookRun.hookGlobal2 = true + } + return nil + }, + }) - fmt.Println("Create queue") - // Fill a queue + // Fill a queue with OnStartup global hooks TasksQueue = task.NewTasksQueue() - // watcher for more verbosity of CreateStartupTasks and - TasksQueue.AddWatcher(&QueueDumperTest{}) TasksQueue.ChangesEnable(true) // Add StartupTasks CreateOnStartupTasks() expectedCount := len(ModuleManager.GetGlobalHooksInOrder(module_manager.OnStartup)) - assert.Equalf(t, expectedCount, TasksQueue.Length(), "queue length is not relevant to global hooks OnStartup", TasksQueue.Length()) + assert.Equal(t, expectedCount, TasksQueue.Length(), "queue length is not equal to count of global 'OnStartup' hooks") // add stop task stopTask := task.NewTask(task.Stop, "stop runner") TasksQueue.Add(stopTask) - fmt.Println("Start task runner") TasksRunner() + assert.True(t, hookRun.hookGlobal1) + assert.True(t, hookRun.hookGlobal2) assert.Equalf(t, 0, TasksQueue.Length(), "%d tasks remain in queue after TasksRunner", TasksQueue.Length()) } - -// Тест заполнения очереди через ModuleManager и его канал EventCh -// Проверяется, что очередь будет заполнена нужным количеством заданий -func TestMain_ModulesEventsHandler(t *testing.T) { - module_manager.EventCh = make(chan module_manager.Event, 1) - ManagersEventsHandlerStopCh = make(chan struct{}, 1) - - // Mock ModuleManager - ModuleManager = &ModuleManagerMock{} - KubeEventsManager = &KubeEventsManagerMock{} - KubeEventsHooks = &KubeEventsHooksControllerMock{} - - fmt.Println("Create queue") - // Fill a queue - TasksQueue = task.NewTasksQueue() - // watcher for more verbosity of CreateStartupTasks and - TasksQueue.AddWatcher(&QueueDumperTest{}) - TasksQueue.ChangesEnable(true) - - assert.Equal(t, 0, TasksQueue.Length()) - - go func(ch chan module_manager.Event) { - ch <- module_manager.Event{ - Type: module_manager.ModulesChanged, - ModulesChanges: []module_manager.ModuleChange{ - { - Name: "test_module_1", - ChangeType: module_manager.Changed, - }, - { - Name: "test_module_2", - ChangeType: module_manager.Changed, - }, - }, - } - ch <- module_manager.Event{ - Type: module_manager.ModulesChanged, - ModulesChanges: []module_manager.ModuleChange{ - { - Name: "test_module_2", - ChangeType: module_manager.Changed, - }, - }, - } - ch <- module_manager.Event{ - Type: module_manager.ModulesChanged, - ModulesChanges: []module_manager.ModuleChange{ - { - Name: "test_module_1", - ChangeType: module_manager.Changed, - }, - }, - } - - ch <- module_manager.Event{ - Type: module_manager.GlobalChanged, - } - }(module_manager.EventCh) - - go ManagersEventsHandler() - - time.Sleep(100 * time.Millisecond) - ManagersEventsHandlerStopCh <- struct{}{} - - expectedCount := 4 // count of ModuleChange in previous go routine - expectedCount += len(ModuleManager.GetGlobalHooksInOrder(module_manager.BeforeAll)) - expectedCount += 1 // DiscoverModulesState task - - assert.Equal(t, expectedCount, TasksQueue.Length()) -} - -func TestMain(m *testing.M) { - - MetricsStorage = metrics_storage.Init() - - os.Exit(m.Run()) -} - -// Тест совместной работы ManagersEventsHandler и TaskRunner. -// один модуль выдаёт ошибку, TaskRunner должен его перезапускать, не запуская другие модули -// проверяется, что модули запускаются по порядку (порядок в runOrder — суффикс имени "__число") -func TestMain_Run_With_InfiniteModuleError(t *testing.T) { - // Настройки задержек при ошибках и пустой очереди, чтобы тест побыстрее завершался. - QueueIsEmptyDelay = 50 * time.Millisecond - FailedHookDelay = 50 * time.Millisecond - FailedModuleDelay = 50 * time.Millisecond - - module_manager.EventCh = make(chan module_manager.Event, 1) - ManagersEventsHandlerStopCh = make(chan struct{}, 1) - - runOrder = []int{} - - // Сделать моки для всего, что нужно для запуска Run - KubeEventsManager = &KubeEventsManagerMock{} - KubeEventsHooks = &KubeEventsHooksControllerMock{} - - helm.Client = MockHelmClient{ - DeleteReleaseErrorsCount: 0, - } - - // Mock ModuleManager - ModuleManager = &ModuleManagerMock{ - BeforeHookErrorsCount: 0, - TestModuleErrorsCount: 10000, - DeleteModuleErrorsCount: 0, - } - - ScheduleManager = &MockScheduleManager{} - - // Создать очередь - fmt.Println("Create queue") - // Fill a queue - TasksQueue = task.NewTasksQueue() - // watcher for more verbosity of CreateStartupTasks and - TasksQueue.AddWatcher(&QueueDumperTest{}) - TasksQueue.ChangesEnable(true) - - Run() - - time.Sleep(1000 * time.Millisecond) - // Stop events handler - ManagersEventsHandlerStopCh <- struct{}{} - // stop tasks runner: add stop task - stopTask := task.NewTask(task.Stop, "stop runner") - TasksQueue.Push(stopTask) - - fmt.Println("wait for queueIsEmptyDelay") - time.Sleep(100 * time.Millisecond) - - assert.True(t, TasksQueue.Length() > 0, "queue is empty with errored module %d", TasksQueue.Length()) - - accum := 0 - for _, ord := range runOrder { - assert.True(t, ord >= accum, "detect unordered execution: '%d' '%d'\n%+v", accum, ord, runOrder) - accum = ord - } - - fmt.Printf("runOrder: %+v", runOrder) -} - -// Тест совместной работы ManagersEventsHandler и TaskRunner. -// Модули и хуки выдают ошибки, TaskRunner должен их перезапускать, не запуская следующие задания. -// Проверяется, что модули и хуки запускаются по порядку (порядок в runOrder — суффикс имени "__число") -func TestMain_Run_With_RecoverableErrors(t *testing.T) { - // Настройки задержек при ошибках и пустой очереди, чтобы тест побыстрее завершался. - QueueIsEmptyDelay = 50 * time.Millisecond - FailedHookDelay = 50 * time.Millisecond - FailedModuleDelay = 50 * time.Millisecond - - module_manager.EventCh = make(chan module_manager.Event, 1) - ManagersEventsHandlerStopCh = make(chan struct{}, 1) - - runOrder = []int{} - - // Сделать моки для всего, что нужно для запуска Run - - helm.Client = MockHelmClient{ - DeleteReleaseErrorsCount: 3, - } - - // Mock ModuleManager - ModuleManager = &ModuleManagerMock{ - BeforeHookErrorsCount: 3, - TestModuleErrorsCount: 6, - DeleteModuleErrorsCount: 2, - } - - ScheduleManager = &MockScheduleManager{} - - fmt.Println("Create queue") - // Fill a queue - TasksQueue = task.NewTasksQueue() - // watcher for more verbosity of CreateStartupTasks and - TasksQueue.AddWatcher(&QueueDumperTest{}) - TasksQueue.ChangesEnable(true) - - Run() - - time.Sleep(1000 * time.Millisecond) - // Stop events handler - ManagersEventsHandlerStopCh <- struct{}{} - // stop tasks runner: add stop task - stopTask := task.NewTask(task.Stop, "stop runner") - TasksQueue.Add(stopTask) - - fmt.Println("wait for queueIsEmptyDelay") - time.Sleep(100 * time.Millisecond) - - assert.Equalf(t, 0, TasksQueue.Length(), "%d tasks remain in queue after TasksRunner", TasksQueue.Length()) - - accum := 0 - for _, ord := range runOrder { - assert.True(t, ord >= accum, "detect unordered execution: '%d' '%d'\n%+v", accum, ord, runOrder) - accum = ord - } - - fmt.Printf("runOrder: %+v", runOrder) -} - -type MockScheduleManager struct { - schedule_manager.ScheduleManager -} - -func (m *MockScheduleManager) Add(crontab string) (string, error) { - fmt.Printf("MockScheduleManager: Add crontab '%s'\n", crontab) - return crontab, nil -} - -func (m *MockScheduleManager) Remove(entryId string) error { - fmt.Printf("MockScheduleManager: Remove crontab '%s'\n", entryId) - return nil -} - -func (m *MockScheduleManager) Run() { - fmt.Printf("MockScheduleManager: Run\n") -} - -// Тесты scheduled_tasks -// Проинициализировать первый раз хуки по расписанию -// Забросить в scheduled канал несколько расписаний -// отключить модуль, забросить GLOBAL изменения, проверить, что хуки пересоздались и остался только глобальный -// забросить в канал расписания, проверить, что выполнится только глобальный хук -func TestMain_ScheduledTasks(t *testing.T) { - - // Настройки задержек при ошибках и пустой очереди, чтобы тест побыстрее завершался. - QueueIsEmptyDelay = 50 * time.Millisecond - FailedHookDelay = 50 * time.Millisecond - FailedModuleDelay = 50 * time.Millisecond - - module_manager.EventCh = make(chan module_manager.Event, 1) - ManagersEventsHandlerStopCh = make(chan struct{}, 1) - - runOrder = []int{} - - helm.Client = MockHelmClient{ - DeleteReleaseErrorsCount: 3, - } - - // Mock ModuleManager - ModuleManager = &ModuleManagerMock{ - BeforeHookErrorsCount: 3, - TestModuleErrorsCount: 6, - DeleteModuleErrorsCount: 2, - } - - // Create ScheduleManager - // Инициализация хуков по расписанию - карта scheduleId → []ScheduleHook - ScheduleManager = &MockScheduleManager{} - schedule_manager.ScheduleCh = make(chan string, 1) - ScheduledHooks = UpdateScheduleHooks(nil) - assert.Equal(t, 4, len(ScheduledHooks), "not enough scheduled hooks") - assert.Equal(t, 3, len(ScheduledHooks.GetHooksForSchedule("*/1 * * * *")), "not enough global scheduled hooks") - - fmt.Println("Create queue") - // Fill a queue - TasksQueue = task.NewTasksQueue() - // watcher for more verbosity of CreateStartupTasks and - TasksQueue.AddWatcher(&QueueDumperTest{}) - TasksQueue.ChangesEnable(true) - - stepCh := make(chan struct{}) - - // обработчик событий от менеджеров — события превращаются в таски и - // добавляются в очередь - go ManagersEventsHandler() - - // TasksRunner запускает задания из очереди - go TasksRunner() - - // EmitScheduleEvents - go func() { - // подождать завершения init тасков - //time.Sleep(300 * time.Millisecond) - schedule_manager.ScheduleCh <- "*/1 * * * *" - time.Sleep(300 * time.Millisecond) - schedule_manager.ScheduleCh <- "*/2 * * * *" - - // удалить хук - delete(scheduledHooks, "scheduled_global_1") - - // GlobalChanged должен привести к пересозданию хранилища хуков по расписанию - time.Sleep(300 * time.Millisecond) - module_manager.EventCh <- module_manager.Event{ - Type: module_manager.GlobalChanged, - } - time.Sleep(100 * time.Millisecond) - stepCh <- struct{}{} - }() - <-stepCh - - // проверка хуков - assert.Equalf(t, 3, len(ScheduledHooks), "bad scheduled hooks count after GlobalChanged: %+v", ScheduledHooks) - - // повторная отправка всех расписаний, в том числе удалённого - go func() { - time.Sleep(300 * time.Millisecond) - schedule_manager.ScheduleCh <- "*/1 * * * *" - time.Sleep(300 * time.Millisecond) - schedule_manager.ScheduleCh <- "*/2 * * * *" - stepCh <- struct{}{} - }() - <-stepCh - - time.Sleep(1000 * time.Millisecond) - - // Stop events handler - ManagersEventsHandlerStopCh <- struct{}{} - - // stop tasks runner: add stop task - stopTask := task.NewTask(task.Stop, "stop runner") - TasksQueue.Add(stopTask) - - fmt.Println("wait for queueIsEmptyDelay") - time.Sleep(100 * time.Millisecond) - - assert.Equalf(t, 0, TasksQueue.Length(), "%d tasks remain in queue after TasksRunner", TasksQueue.Length()) - - // TODO надо этот order переделать, чтобы были не чиселки, а лог выполнения модулей/хуков - accum := 0 - for _, ord := range runOrder { - assert.True(t, ord >= accum, "detect unordered execution: '%d' '%d'\n%+v", accum, ord, runOrder) - accum = ord - } - - fmt.Printf("runOrder: %+v", runOrder) -} - -// Тесты запускаются уже с flag.Parsed(), поэтому glog ничего не пишет -func TestGlog(t *testing.T) { - t.SkipNow() - os.Setenv("RLOG_LOG_LEVEL", "DEBUG") - rlog.UpdateEnv() - - rlog.Info("start TestGlog") - glog.Warning("test warngin from glog") - - flag.Set("", "") - //flag.CommandLine.Parse([]string{}) - glog.Warning("test warngin from glog after Parse") - rlog.Info("stop TestGlog") - - time.Sleep(1 * time.Second) - - t.Error("Error call to get stdout") -} - -// Dump global hooks and modules and modules hooks info -func TestDumpModuleManagerInfo(t *testing.T) { - t.SkipNow() - rlog.Debugf("=== START ModuleManager Dump ===") - bindings := []module_manager.BindingType{module_manager.OnStartup, module_manager.BeforeHelm, module_manager.AfterHelm, - module_manager.AfterDeleteHelm, module_manager.BeforeAll, module_manager.AfterAll, module_manager.Schedule, module_manager.KubeEvents, - } - rlog.Debugf(" GlobalHooks") - - for _, binding := range bindings { - ghNames := ModuleManager.GetGlobalHooksInOrder(binding) - if len(ghNames) > 0 { - rlog.Debugf(" %s:", binding) - for idx, ghName := range ghNames { - gh, _ := ModuleManager.GetGlobalHook(ghName) - rlog.Debugf("%d. %s %s %s safe: %s, bind: %+v", idx, ghName, gh.Name, path.Base(gh.Path), gh.SafeName()) - } - } - } - - rlog.Debugf(" Modules and module hooks") - mNames := ModuleManager.GetModuleNamesInOrder() - for idx, mName := range mNames { - rlog.Debugf("%d. %s", idx, mName) - for _, binding := range bindings { - mhNames, _ := ModuleManager.GetModuleHooksInOrder(mName, binding) - if len(mhNames) > 0 { - rlog.Debugf(" %s:", binding) - for _, mhName := range mhNames { - mh, _ := ModuleManager.GetModuleHook(mhName) - rlog.Debugf(" %s '%s' safe:'%s' %s module: '%s' safe:'%s'", mhName, mh.Name, mh.SafeName(), path.Base(mh.Path), mh.Module.Name, mh.Module.SafeName()) - } - } - } - } - rlog.Debugf("=== END ModuleManager Dump ===") -} diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 345e5d93..42f9cdb2 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -91,7 +91,7 @@ func (helm *CliHelm) Cmd(args ...string) (stdout string, stderr string, err erro var stderrBuf bytes.Buffer cmd.Stderr = &stderrBuf - err = executor.Run(cmd, true) + err = executor.Run(cmd) stdout = strings.TrimSpace(stdoutBuf.String()) stderr = strings.TrimSpace(stderrBuf.String()) diff --git a/pkg/module_manager/global_hook.go b/pkg/module_manager/global_hook.go new file mode 100644 index 00000000..e04223eb --- /dev/null +++ b/pkg/module_manager/global_hook.go @@ -0,0 +1,298 @@ +package module_manager + +import ( + "encoding/json" + "fmt" + "path/filepath" + + "github.com/romana/rlog" + + hook2 "github.com/flant/shell-operator/pkg/hook" + utils_data "github.com/flant/shell-operator/pkg/utils/data" + + "github.com/flant/addon-operator/pkg/utils" +) + +type GlobalHook struct { + *CommonHook + Config *GlobalHookConfig +} +var _ Hook = &GlobalHook{} + +func NewGlobalHook(name, path string) *GlobalHook { + return &GlobalHook{ + CommonHook: &CommonHook{ + Name: name, + Path: path, + }, + Config: &GlobalHookConfig{}, + } +} + +func (g *GlobalHook) WithConfig(configOutput []byte) (err error) { + err = g.Config.LoadAndValidate(configOutput) + if err != nil { + return fmt.Errorf("load global hook '%s' config: %s\nhook --config output: %s", g.Name, err.Error(), configOutput) + } + return nil +} + +func (g *GlobalHook) Order(binding BindingType) float64 { + if g.Config.HasBinding(binding) { + switch binding { + case BeforeAll: + return g.Config.BeforeAll.Order + case AfterAll: + return g.Config.AfterAll.Order + } + } + return 0.0 +} + +type globalValuesMergeResult struct { + // Global values with the root "global" key. + Values utils.Values + // Global values under the root "global" key. + GlobalValues map[string]interface{} + // Original values patch argument. + ValuesPatch utils.ValuesPatch + // Whether values changed after applying patch. + ValuesChanged bool +} + +func (h *GlobalHook) handleGlobalValuesPatch(currentValues utils.Values, valuesPatch utils.ValuesPatch) (*globalValuesMergeResult, error) { + acceptableKey := "global" + + if err := utils.ValidateHookValuesPatch(valuesPatch, acceptableKey); err != nil { + return nil, fmt.Errorf("merge global values failed: %s", err) + } + + newValuesRaw, valuesChanged, err := utils.ApplyValuesPatch(currentValues, valuesPatch) + if err != nil { + return nil, fmt.Errorf("merge global values failed: %s", err) + } + + result := &globalValuesMergeResult{ + Values: utils.Values{acceptableKey: make(map[string]interface{})}, + ValuesChanged: valuesChanged, + ValuesPatch: valuesPatch, + } + + if globalValuesRaw, hasKey := newValuesRaw[acceptableKey]; hasKey { + globalValues, ok := globalValuesRaw.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("expected map at key '%s', got:\n%s", acceptableKey, utils_data.YamlToString(globalValuesRaw)) + } + + result.Values[acceptableKey] = globalValues + result.GlobalValues = globalValues + } + + return result, nil +} + +func (h *GlobalHook) run(bindingType BindingType, context []BindingContext) error { + rlog.Infof("Running global hook '%s' binding '%s' ...", h.Name, bindingType) + + // Convert context for version + versionedContext := make([]interface{}, 0, len(context)) + for _, c := range context { + versionedContext = append(versionedContext, hook2.ConvertBindingContext(h.Config.Version, c.BindingContext)) + } + + globalHookExecutor := NewHookExecutor(h, versionedContext) + patches, err := globalHookExecutor.Run() + if err != nil { + return fmt.Errorf("global hook '%s' failed: %s", h.Name, err) + } + + configValuesPatch, has := patches[utils.ConfigMapPatch] + if has && configValuesPatch != nil { + preparedConfigValues := utils.MergeValues( + utils.Values{"global": map[string]interface{}{}}, + h.moduleManager.kubeGlobalConfigValues, + ) + + configValuesPatchResult, err := h.handleGlobalValuesPatch(preparedConfigValues, *configValuesPatch) + if err != nil { + return fmt.Errorf("global hook '%s': kube config global values update error: %s", h.Name, err) + } + + if configValuesPatchResult.ValuesChanged { + err := h.moduleManager.kubeConfigManager.SetKubeGlobalValues(configValuesPatchResult.Values) + if err != nil { + rlog.Debugf("Global hook '%s' kube config global values stay unchanged:\n%s", utils.ValuesToString(h.moduleManager.kubeGlobalConfigValues)) + return fmt.Errorf("global hook '%s': set kube config failed: %s", h.Name, err) + } + + h.moduleManager.kubeGlobalConfigValues = configValuesPatchResult.Values + rlog.Debugf("Global hook '%s': kube config global values updated:\n%s", h.Name, utils.ValuesToString(h.moduleManager.kubeGlobalConfigValues)) + } + } + + valuesPatch, has := patches[utils.MemoryValuesPatch] + if has && valuesPatch != nil { + valuesPatchResult, err := h.handleGlobalValuesPatch(h.values(), *valuesPatch) + if err != nil { + return fmt.Errorf("global hook '%s': dynamic global values update error: %s", h.Name, err) + } + if valuesPatchResult.ValuesChanged { + h.moduleManager.globalDynamicValuesPatches = utils.AppendValuesPatch(h.moduleManager.globalDynamicValuesPatches, valuesPatchResult.ValuesPatch) + rlog.Debugf("Global hook '%s': global values updated:\n%s", h.Name, utils.ValuesToString(h.values())) + } + } + + return nil +} + +// PrepareTmpFilesForHookRun creates temporary files for hook and returns environment variables with paths +func (h *GlobalHook) PrepareTmpFilesForHookRun(context interface{}) (tmpFiles map[string]string, err error) { + tmpFiles = make(map[string]string, 0) + + tmpFiles["CONFIG_VALUES_PATH"], err = h.prepareConfigValuesJsonFile() + if err != nil { + return + } + + tmpFiles["VALUES_PATH"], err = h.prepareValuesJsonFile() + if err != nil { + return + } + + tmpFiles["BINDING_CONTEXT_PATH"], err = h.prepareBindingContextJsonFile(context) + if err != nil { + return + } + + tmpFiles["CONFIG_VALUES_JSON_PATCH_PATH"], err = h.prepareConfigValuesJsonPatchFile() + if err != nil { + return + } + + tmpFiles["VALUES_JSON_PATCH_PATH"], err = h.prepareValuesJsonPatchFile() + if err != nil { + return + } + + return +} + + +func (h *GlobalHook) configValues() utils.Values { + return utils.MergeValues( + utils.Values{"global": map[string]interface{}{}}, + h.moduleManager.kubeGlobalConfigValues, + ) +} + +func (h *GlobalHook) values() utils.Values { + var err error + + res := utils.MergeValues( + utils.Values{"global": map[string]interface{}{}}, + h.moduleManager.globalCommonStaticValues, + h.moduleManager.kubeGlobalConfigValues, + ) + + // Invariant: do not store patches that does not apply + // Give user error for patches early, after patch receive + for _, patch := range h.moduleManager.globalDynamicValuesPatches { + res, _, err = utils.ApplyValuesPatch(res, patch) + if err != nil { + panic(err) + } + } + + return res +} + +func (h *GlobalHook) prepareConfigValuesYamlFile() (string, error) { + values := h.configValues() + + data := utils.MustDump(utils.DumpValuesYaml(values)) + path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("global-hook-%s-config-values.yaml", h.SafeName())) + err := dumpData(path, data) + if err != nil { + return "", err + } + + rlog.Debugf("Prepared global hook %s config values:\n%s", h.Name, utils.ValuesToString(values)) + + return path, nil +} + +func (h *GlobalHook) prepareConfigValuesJsonFile() (string, error) { + values := h.configValues() + + data := utils.MustDump(utils.DumpValuesJson(values)) + path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("global-hook-%s-config-values.json", h.SafeName())) + err := dumpData(path, data) + if err != nil { + return "", err + } + + rlog.Debugf("Prepared global hook %s config values:\n%s", h.Name, utils.ValuesToString(values)) + + return path, nil +} + +func (h *GlobalHook) prepareValuesYamlFile() (string, error) { + values := h.values() + + data := utils.MustDump(utils.DumpValuesYaml(values)) + path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("global-hook-%s-values.yaml", h.SafeName())) + err := dumpData(path, data) + if err != nil { + return "", err + } + + rlog.Debugf("Prepared global hook %s values:\n%s", h.Name, utils.ValuesToString(values)) + + return path, nil +} + +func (h *GlobalHook) prepareValuesJsonFile() (string, error) { + values := h.values() + + data := utils.MustDump(utils.DumpValuesJson(values)) + path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("global-hook-%s-values.json", h.SafeName())) + err := dumpData(path, data) + if err != nil { + return "", err + } + + rlog.Debugf("Prepared global hook %s values:\n%s", h.Name, utils.ValuesToString(values)) + + return path, nil +} + +func (h *GlobalHook) prepareBindingContextJsonFile(context interface{}) (string, error) { + data, _ := json.Marshal(context) + //data := utils.MustDump(utils.DumpValuesJson(context)) + path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("global-hook-%s-binding-context.json", h.SafeName())) + err := dumpData(path, data) + if err != nil { + return "", err + } + + rlog.Debugf("Prepared global hook %s binding context:\n%s", h.Name, utils_data.YamlToString(context)) + + return path, nil +} + + +func (h *GlobalHook) prepareConfigValuesJsonPatchFile() (string, error) { + path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("%s.global-hook-config-values.json-patch", h.SafeName())) + if err := CreateEmptyWritableFile(path); err != nil { + return "", err + } + return path, nil +} + +func (h *GlobalHook) prepareValuesJsonPatchFile() (string, error) { + path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("%s.global-hook-values.json-patch", h.SafeName())) + if err := CreateEmptyWritableFile(path); err != nil { + return "", err + } + return path, nil +} diff --git a/pkg/module_manager/global_hook_config.go b/pkg/module_manager/global_hook_config.go new file mode 100644 index 00000000..d31f21d7 --- /dev/null +++ b/pkg/module_manager/global_hook_config.go @@ -0,0 +1,214 @@ +package module_manager + +import ( + "encoding/json" + "fmt" + + "github.com/go-openapi/spec" + + hook_config "github.com/flant/shell-operator/pkg/hook" + "github.com/flant/shell-operator/pkg/hook/config" +) + +// GlobalHookConfig is a structure with versioned hook configuration +type GlobalHookConfig struct { + hook_config.HookConfig + + // versioned raw config values + GlobalV0 *GlobalHookConfigV0 + GlobalV1 *GlobalHookConfigV0 + + // effective config values + BeforeAll *BeforeAllConfig + AfterAll *AfterAllConfig +} + +type BeforeAllConfig struct { + hook_config.CommonBindingConfig + Order float64 +} + +type AfterAllConfig struct { + hook_config.CommonBindingConfig + Order float64 +} + +type GlobalHookConfigV0 struct { + BeforeAll interface{} `json:"beforeAll"` + AfterAll interface{} `json:"afterAll"` +} + +func GetGlobalHookConfigSchema(version string) *spec.Schema { + globalHookVersion := "global-hook-"+version + if _, ok := config.Schemas[globalHookVersion]; !ok { + schema := config.Schemas[version] + switch version { + case "v1": + // add beforeAll and afterAll properties + schema += ` + beforeAll: + type: integer + example: 10 + afterAll: + type: integer + example: 10 +` + case "v0": + // add beforeAll and afterAll properties + schema += ` + beforeAll: + type: integer + example: 10 + afterAll: + type: integer + example: 10 +` + } + config.Schemas[globalHookVersion] = schema + } + + return config.GetSchema(globalHookVersion) +} + +// LoadAndValidate loads config from bytes and validate it. Returns multierror. +func (c *GlobalHookConfig) LoadAndValidate(data []byte) error { + vu := config.NewDefaultVersionedUntyped() + err := vu.Load(data) + if err != nil { + return err + } + + err = config.ValidateConfig(vu.Obj, GetGlobalHookConfigSchema(vu.Version), "") + if err != nil { + return err + } + + c.Version = vu.Version + + err = c.HookConfig.ConvertAndCheck(data) + if err != nil { + return err + } + + err = c.ConvertAndCheck(data) + if err != nil { + return err + } + + return nil +} + +func (c *GlobalHookConfig) ConvertAndCheck(data []byte) error { + switch c.Version { + case "v0": + configV0 := &GlobalHookConfigV0{} + err := json.Unmarshal(data, configV0) + if err != nil { + return fmt.Errorf("unmarshal GlobalHookConfig version 0: %s", err) + } + c.GlobalV0 = configV0 + err = c.ConvertAndCheckV0() + if err != nil { + return err + } + case "v1": + configV1 := &GlobalHookConfigV0{} + err := json.Unmarshal(data, configV1) + if err != nil { + return fmt.Errorf("unmarshal GlobalHookConfig v1: %s", err) + } + c.GlobalV1 = configV1 + err = c.ConvertAndCheckV1() + if err != nil { + return err + } + default: + // NOTE: this should not happen + return fmt.Errorf("version '%s' is unsupported", c.Version) + } + + return nil +} + +func (c *GlobalHookConfig) ConvertAndCheckV0() (err error) { + c.BeforeAll, err = c.ConvertBeforeAll(c.GlobalV0.BeforeAll) + if err != nil { + return err + } + + c.AfterAll, err = c.ConvertAfterAll(c.GlobalV0.AfterAll) + if err != nil { + return err + } + + return nil +} + +func (c *GlobalHookConfig) ConvertAndCheckV1() (err error) { + c.BeforeAll, err = c.ConvertBeforeAll(c.GlobalV1.BeforeAll) + if err != nil { + return err + } + + c.AfterAll, err = c.ConvertAfterAll(c.GlobalV1.AfterAll) + if err != nil { + return err + } + + return nil +} + +func (c *GlobalHookConfig) ConvertBeforeAll(value interface{}) (*BeforeAllConfig, error) { + floatValue, err := hook_config.ConvertFloatForBinding(value, "beforeAll") + if err != nil || floatValue == nil { + return nil, err + } + + res := &BeforeAllConfig{} + res.ConfigName = ContextBindingType[BeforeAll] + res.Order = *floatValue + return res, nil +} + +func (c *GlobalHookConfig) ConvertAfterAll(value interface{}) (*AfterAllConfig, error) { + floatValue, err := hook_config.ConvertFloatForBinding(value, "afterAll") + if err != nil || floatValue == nil { + return nil, err + } + + res := &AfterAllConfig{} + res.ConfigName = ContextBindingType[AfterAll] + res.Order = *floatValue + return res, nil +} + +func (c *GlobalHookConfig) Bindings() []BindingType { + res := []BindingType{} + + for _, binding := range []BindingType{OnStartup, Schedule, KubeEvents} { + if c.HookConfig.HasBinding(ShOpBindingType[binding]) { + res = append(res, binding) + } + } + + for _, binding := range []BindingType{BeforeAll, AfterAll} { + if c.HasBinding(binding) { + res = append(res, binding) + } + } + + return res +} + +func (c *GlobalHookConfig) HasBinding(binding BindingType) bool { + if c.HookConfig.HasBinding(ShOpBindingType[binding]) { + return true + } + switch binding { + case BeforeAll: + return c.BeforeAll != nil + case AfterAll: + return c.AfterAll != nil + } + return false +} diff --git a/pkg/module_manager/global_hook_config_test.go b/pkg/module_manager/global_hook_config_test.go new file mode 100644 index 00000000..714ced57 --- /dev/null +++ b/pkg/module_manager/global_hook_config_test.go @@ -0,0 +1,48 @@ +package module_manager + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_GlobalHook_Config_v0_v1(t *testing.T) { + var err error + var config *GlobalHookConfig + + tests := []struct { + name string + hookName string + data string + assertion func() + }{ + { + "load v0 config", + "hook_v0", + `{"onStartup":10, "schedule":[{"crontab":"*/5 * * * * *"}], "beforeAll":10}`, + func() { + if !assert.NoError(t, err) { + t.FailNow() + } + }, + }, + { + "load v1 config", + "hook_v1", + `{"configVersion": "v1", "onStartup":10, "kubernetes":[{"kind":"Pod", "watchEvent":["Added"]}], "afterAll":10}`, + func() { + if !assert.NoError(t, err) { + t.FailNow() + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config = &GlobalHookConfig{} + err = config.LoadAndValidate([]byte(test.data)) + test.assertion() + }) + } +} diff --git a/pkg/module_manager/global_hook_test.go b/pkg/module_manager/global_hook_test.go new file mode 100644 index 00000000..a6f1bdad --- /dev/null +++ b/pkg/module_manager/global_hook_test.go @@ -0,0 +1,7 @@ +package module_manager + +import "testing" + +func Test_GlobalHook_Creating(t *testing.T) { + +} diff --git a/pkg/module_manager/hook.go b/pkg/module_manager/hook.go index 80ab4f01..c8d47b42 100644 --- a/pkg/module_manager/hook.go +++ b/pkg/module_manager/hook.go @@ -1,40 +1,24 @@ package module_manager import ( - "encoding/json" "fmt" "os" - "os/exec" "path/filepath" - "strings" + "sort" "github.com/kennygrant/sanitize" "github.com/romana/rlog" - "github.com/flant/shell-operator/pkg/executor" - "github.com/flant/shell-operator/pkg/kube_events_manager" - "github.com/flant/shell-operator/pkg/schedule_manager" - utils_data "github.com/flant/shell-operator/pkg/utils/data" - - "github.com/flant/addon-operator/pkg/helm" - "github.com/flant/addon-operator/pkg/utils" + utils_file "github.com/flant/shell-operator/pkg/utils/file" ) -type GlobalHook struct { - *CommonHook - Config *GlobalHookConfig -} - -type ModuleHook struct { - *CommonHook - Module *Module - Config *ModuleHookConfig -} - type Hook interface { + WithModuleManager(moduleManager *MainModuleManager) + WithConfig(configOutput []byte) (err error) GetName() string GetPath() string - PrepareTmpFilesForHookRun(context []BindingContext) (map[string]string, error) + PrepareTmpFilesForHookRun(context interface{}) (map[string]string, error) + Order(binding BindingType) float64 } type CommonHook struct { @@ -43,637 +27,159 @@ type CommonHook struct { // The absolute path of the executable file. Path string - Bindings []BindingType - OrderByBinding map[BindingType]float64 - moduleManager *MainModuleManager } -type GlobalHookConfig struct { - HookConfig - BeforeAll interface{} `json:"beforeAll"` - AfterAll interface{} `json:"afterAll"` +func (c *CommonHook) WithModuleManager(moduleManager *MainModuleManager) { + c.moduleManager = moduleManager } -type ModuleHookConfig struct { - HookConfig - BeforeHelm interface{} `json:"beforeHelm"` - AfterHelm interface{} `json:"afterHelm"` - AfterDeleteHelm interface{} `json:"afterDeleteHelm"` -} - -type HookConfig struct { - OnStartup interface{} `json:"onStartup"` - Schedule []schedule_manager.ScheduleConfig `json:"schedule"` - OnKubernetesEvent []kube_events_manager.OnKubernetesEventConfig `json:"onKubernetesEvent"` -} - -func NewGlobalHook(name, path string, config *GlobalHookConfig, mm *MainModuleManager) *GlobalHook { - globalHook := &GlobalHook{} - globalHook.CommonHook = NewHook(name, path, mm) - globalHook.Config = config - return globalHook -} - -func NewHook(name, path string, mm *MainModuleManager) *CommonHook { - hook := &CommonHook{} - hook.moduleManager = mm - hook.Name = name - hook.Path = path - hook.OrderByBinding = make(map[BindingType]float64) - return hook -} - -func NewModuleHook(name, path string, config *ModuleHookConfig, mm *MainModuleManager) *ModuleHook { - moduleHook := &ModuleHook{} - moduleHook.CommonHook = NewHook(name, path, mm) - moduleHook.Config = config - return moduleHook -} - -func (mm *MainModuleManager) registerGlobalHook(name, path string, config *GlobalHookConfig) (err error) { - var ok bool - globalHook := NewGlobalHook(name, path, config, mm) - - if config.BeforeAll != nil { - globalHook.Bindings = append(globalHook.Bindings, BeforeAll) - if globalHook.OrderByBinding[BeforeAll], ok = config.BeforeAll.(float64); !ok { - return fmt.Errorf("unsuported value '%v' for binding '%s'", config.BeforeAll, BeforeAll) - } - mm.globalHooksOrder[BeforeAll] = append(mm.globalHooksOrder[BeforeAll], globalHook) - } - - if config.AfterAll != nil { - globalHook.Bindings = append(globalHook.Bindings, AfterAll) - if globalHook.OrderByBinding[AfterAll], ok = config.AfterAll.(float64); !ok { - return fmt.Errorf("unsuported value '%v' for binding '%s'", config.AfterAll, AfterAll) - } - mm.globalHooksOrder[AfterAll] = append(mm.globalHooksOrder[AfterAll], globalHook) - } - - if config.OnStartup != nil { - globalHook.Bindings = append(globalHook.Bindings, OnStartup) - if globalHook.OrderByBinding[OnStartup], ok = config.OnStartup.(float64); !ok { - return fmt.Errorf("unsuported value '%v' for binding '%s'", config.OnStartup, OnStartup) - } - mm.globalHooksOrder[OnStartup] = append(mm.globalHooksOrder[OnStartup], globalHook) - } - - if len(config.Schedule) != 0 { - globalHook.Bindings = append(globalHook.Bindings, Schedule) - mm.globalHooksOrder[Schedule] = append(mm.globalHooksOrder[Schedule], globalHook) - } - - if len(config.OnKubernetesEvent) != 0 { - globalHook.Bindings = append(globalHook.Bindings, KubeEvents) - mm.globalHooksOrder[KubeEvents] = append(mm.globalHooksOrder[KubeEvents], globalHook) - } - - mm.globalHooksByName[name] = globalHook - - return nil -} - -func (mm *MainModuleManager) registerModuleHook(moduleName, name, path string, config *ModuleHookConfig) (err error) { - var ok bool - moduleHook := NewModuleHook(name, path, config, mm) - - if moduleHook.Module, err = mm.GetModule(moduleName); err != nil { - return err - } - - if config.BeforeHelm != nil { - moduleHook.Bindings = append(moduleHook.Bindings, BeforeHelm) - if moduleHook.OrderByBinding[BeforeHelm], ok = config.BeforeHelm.(float64); !ok { - return fmt.Errorf("unsuported value '%v' for binding '%s'", config.BeforeHelm, BeforeHelm) - } - - mm.addModulesHooksOrderByName(moduleName, BeforeHelm, moduleHook) - } - - if config.AfterHelm != nil { - moduleHook.Bindings = append(moduleHook.Bindings, AfterHelm) - if moduleHook.OrderByBinding[AfterHelm], ok = config.AfterHelm.(float64); !ok { - return fmt.Errorf("unsuported value '%v' for binding '%s'", config.AfterHelm, AfterHelm) - } - mm.addModulesHooksOrderByName(moduleName, AfterHelm, moduleHook) - } - - if config.AfterDeleteHelm != nil { - moduleHook.Bindings = append(moduleHook.Bindings, AfterDeleteHelm) - if moduleHook.OrderByBinding[AfterDeleteHelm], ok = config.AfterDeleteHelm.(float64); !ok { - return fmt.Errorf("unsuported value '%v' for binding '%s'", config.AfterDeleteHelm, AfterDeleteHelm) - } - mm.addModulesHooksOrderByName(moduleName, AfterDeleteHelm, moduleHook) - } - - if config.OnStartup != nil { - moduleHook.Bindings = append(moduleHook.Bindings, OnStartup) - if moduleHook.OrderByBinding[OnStartup], ok = config.OnStartup.(float64); !ok { - return fmt.Errorf("unsuported value '%v' for binding '%s'", config.OnStartup, OnStartup) - } - mm.addModulesHooksOrderByName(moduleName, OnStartup, moduleHook) - } - - if len(config.Schedule) != 0 { - moduleHook.Bindings = append(moduleHook.Bindings, Schedule) - mm.addModulesHooksOrderByName(moduleName, Schedule, moduleHook) - } - - if len(config.OnKubernetesEvent) != 0 { - moduleHook.Bindings = append(moduleHook.Bindings, KubeEvents) - mm.addModulesHooksOrderByName(moduleName, KubeEvents, moduleHook) - } - - return nil +func (h *CommonHook) SafeName() string { + return sanitize.BaseName(h.Name) } -func (mm *MainModuleManager) addModulesHooksOrderByName(moduleName string, bindingType BindingType, moduleHook *ModuleHook) { - if mm.modulesHooksOrderByName[moduleName] == nil { - mm.modulesHooksOrderByName[moduleName] = make(map[BindingType][]*ModuleHook) - } - mm.modulesHooksOrderByName[moduleName][bindingType] = append(mm.modulesHooksOrderByName[moduleName][bindingType], moduleHook) +func (h *CommonHook) GetName() string { + return h.Name } -func (mm *MainModuleManager) removeModuleHooks(moduleName string) { - delete(mm.modulesHooksOrderByName, moduleName) +func (h *CommonHook) GetPath() string { + return h.Path } -type globalValuesMergeResult struct { - // Global values with the root "global" key. - Values utils.Values - // Global values under the root "global" key. - GlobalValues map[string]interface{} - // Original values patch argument. - ValuesPatch utils.ValuesPatch - // Whether values changed after applying patch. - ValuesChanged bool -} - -func (h *GlobalHook) handleGlobalValuesPatch(currentValues utils.Values, valuesPatch utils.ValuesPatch) (*globalValuesMergeResult, error) { - acceptableKey := "global" +func SearchGlobalHooks(hooksDir string) (hooks []*GlobalHook, err error) { + rlog.Debug("INIT: search global hooks...") - if err := validateHookValuesPatch(valuesPatch, acceptableKey); err != nil { - return nil, fmt.Errorf("merge global values failed: %s", err) + if _, err := os.Stat(hooksDir); os.IsNotExist(err) { + return nil, nil } - newValuesRaw, valuesChanged, err := utils.ApplyValuesPatch(currentValues, valuesPatch) + hooksRelativePaths, err := utils_file.RecursiveGetExecutablePaths(hooksDir) if err != nil { - return nil, fmt.Errorf("merge global values failed: %s", err) - } - - result := &globalValuesMergeResult{ - Values: utils.Values{acceptableKey: make(map[string]interface{})}, - ValuesChanged: valuesChanged, - ValuesPatch: valuesPatch, - } - - if globalValuesRaw, hasKey := newValuesRaw[acceptableKey]; hasKey { - globalValues, ok := globalValuesRaw.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("expected map at key '%s', got:\n%s", acceptableKey, utils_data.YamlToString(globalValuesRaw)) - } - - result.Values[acceptableKey] = globalValues - result.GlobalValues = globalValues + return nil, err } - return result, nil -} - -func (h *GlobalHook) run(bindingType BindingType, context []BindingContext) error { - rlog.Infof("Running global hook '%s' binding '%s' ...", h.Name, bindingType) - - globalHookExecutor := NewHookExecutor(h, context) - patches, err := globalHookExecutor.Run() - if err != nil { - return fmt.Errorf("global hook '%s' failed: %s", h.Name, err) - } + hooks = make([]*GlobalHook, 0) - configValuesPatch, has := patches[utils.ConfigMapPatch] - if has && configValuesPatch != nil { - preparedConfigValues := utils.MergeValues( - utils.Values{"global": map[string]interface{}{}}, - h.moduleManager.kubeGlobalConfigValues, - ) + // sort hooks by path + sort.Strings(hooksRelativePaths) + rlog.Debugf(" Hook paths: %+v", hooksRelativePaths) - configValuesPatchResult, err := h.handleGlobalValuesPatch(preparedConfigValues, *configValuesPatch) + for _, hookPath := range hooksRelativePaths { + hookName, err := filepath.Rel(hooksDir, hookPath) if err != nil { - return fmt.Errorf("global hook '%s': kube config global values update error: %s", h.Name, err) + return nil, err } - if configValuesPatchResult.ValuesChanged { - if err := h.moduleManager.kubeConfigManager.SetKubeGlobalValues(configValuesPatchResult.Values); err != nil { - rlog.Debugf("Global hook '%s' kube config global values stay unchanged:\n%s", utils.ValuesToString(h.moduleManager.kubeGlobalConfigValues)) - return fmt.Errorf("global hook '%s': set kube config failed: %s", h.Name, err) - } + rlog.Infof("INIT: global hook '%s'", hookName) - h.moduleManager.kubeGlobalConfigValues = configValuesPatchResult.Values - rlog.Debugf("Global hook '%s': kube config global values updated:\n%s", h.Name, utils.ValuesToString(h.moduleManager.kubeGlobalConfigValues)) - } - } + globalHook := NewGlobalHook(hookName, hookPath) - valuesPatch, has := patches[utils.MemoryValuesPatch] - if has && valuesPatch != nil { - valuesPatchResult, err := h.handleGlobalValuesPatch(h.values(), *valuesPatch) - if err != nil { - return fmt.Errorf("global hook '%s': dynamic global values update error: %s", h.Name, err) - } - if valuesPatchResult.ValuesChanged { - h.moduleManager.globalDynamicValuesPatches = utils.AppendValuesPatch(h.moduleManager.globalDynamicValuesPatches, valuesPatchResult.ValuesPatch) - rlog.Debugf("Global hook '%s': global values updated:\n%s", h.Name, utils.ValuesToString(h.values())) - } + hooks = append(hooks, globalHook) } - return nil + return } -// PrepareTmpFilesForHookRun creates temporary files for hook and returns environment variables with paths -func (h *GlobalHook) PrepareTmpFilesForHookRun(context []BindingContext) (tmpFiles map[string]string, err error) { - tmpFiles = make(map[string]string, 0) +func SearchModuleHooks(module *Module) (hooks []*ModuleHook, err error) { + rlog.Infof("INIT: module '%s' hooks ...", module.Name) - tmpFiles["CONFIG_VALUES_PATH"], err = h.prepareConfigValuesJsonFile() - if err != nil { - return + hooksDir := filepath.Join(module.Path, "hooks") + if _, err := os.Stat(hooksDir); os.IsNotExist(err) { + return nil, nil } - tmpFiles["VALUES_PATH"], err = h.prepareValuesJsonFile() + hooksRelativePaths, err := utils_file.RecursiveGetExecutablePaths(hooksDir) if err != nil { - return - } - - if len(context) > 0 { - tmpFiles["BINDING_CONTEXT_PATH"], err = h.prepareBindingContextJsonFile(context) - if err != nil { - return - } + return nil, err } - tmpFiles["CONFIG_VALUES_JSON_PATCH_PATH"], err = h.prepareConfigValuesJsonPatchFile() - if err != nil { - return - } + hooks = make([]*ModuleHook, 0) - tmpFiles["VALUES_JSON_PATCH_PATH"], err = h.prepareValuesJsonPatchFile() - if err != nil { - return - } + // sort hooks by path + sort.Strings(hooksRelativePaths) + rlog.Debugf(" Hook paths: %+v", hooksRelativePaths) - return -} - - -func (h *GlobalHook) configValues() utils.Values { - return utils.MergeValues( - utils.Values{"global": map[string]interface{}{}}, - h.moduleManager.kubeGlobalConfigValues, - ) -} - -func (h *GlobalHook) values() utils.Values { - var err error - - res := utils.MergeValues( - utils.Values{"global": map[string]interface{}{}}, - h.moduleManager.globalCommonStaticValues, - h.moduleManager.kubeGlobalConfigValues, - ) - - // Invariant: do not store patches that does not apply - // Give user error for patches early, after patch receive - for _, patch := range h.moduleManager.globalDynamicValuesPatches { - res, _, err = utils.ApplyValuesPatch(res, patch) + for _, hookPath := range hooksRelativePaths { + hookName, err := filepath.Rel(filepath.Dir(module.Path), hookPath) if err != nil { - panic(err) + return nil, err } - } - - return res -} - -func (h *GlobalHook) prepareConfigValuesYamlFile() (string, error) { - values := h.configValues() - - data := utils.MustDump(utils.DumpValuesYaml(values)) - path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("global-hook-%s-config-values.yaml", h.SafeName())) - err := dumpData(path, data) - if err != nil { - return "", err - } - - rlog.Debugf("Prepared global hook %s config values:\n%s", h.Name, utils.ValuesToString(values)) - - return path, nil -} - -func (h *GlobalHook) prepareConfigValuesJsonFile() (string, error) { - values := h.configValues() - - data := utils.MustDump(utils.DumpValuesJson(values)) - path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("global-hook-%s-config-values.json", h.SafeName())) - err := dumpData(path, data) - if err != nil { - return "", err - } - - rlog.Debugf("Prepared global hook %s config values:\n%s", h.Name, utils.ValuesToString(values)) - - return path, nil -} - -func (h *GlobalHook) prepareValuesYamlFile() (string, error) { - values := h.values() - data := utils.MustDump(utils.DumpValuesYaml(values)) - path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("global-hook-%s-values.yaml", h.SafeName())) - err := dumpData(path, data) - if err != nil { - return "", err - } - - rlog.Debugf("Prepared global hook %s values:\n%s", h.Name, utils.ValuesToString(values)) - - return path, nil -} - -func (h *GlobalHook) prepareValuesJsonFile() (string, error) { - values := h.values() - - data := utils.MustDump(utils.DumpValuesJson(values)) - path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("global-hook-%s-values.json", h.SafeName())) - err := dumpData(path, data) - if err != nil { - return "", err - } - - rlog.Debugf("Prepared global hook %s values:\n%s", h.Name, utils.ValuesToString(values)) + rlog.Infof("INIT: hook '%s' ...", hookName) - return path, nil -} + moduleHook := NewModuleHook(hookName, hookPath) + moduleHook.WithModule(module) -func (h *GlobalHook) prepareBindingContextJsonFile(context []BindingContext) (string, error) { - data, _ := json.Marshal(context) - //data := utils.MustDump(utils.DumpValuesJson(context)) - path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("global-hook-%s-binding-context.json", h.SafeName())) - err := dumpData(path, data) - if err != nil { - return "", err + hooks = append(hooks, moduleHook) } - rlog.Debugf("Prepared global hook %s binding context:\n%s", h.Name, utils_data.YamlToString(context)) - - return path, nil -} - -type moduleValuesMergeResult struct { - // global values with root ModuleValuesKey key - Values utils.Values - // global values under root ModuleValuesKey key - ModuleValues map[string]interface{} - ModuleValuesKey string - ValuesPatch utils.ValuesPatch - ValuesChanged bool -} - -func (h *CommonHook) SafeName() string { - return sanitize.BaseName(h.Name) -} - -func (h *CommonHook) GetName() string { - return h.Name -} - -func (h *CommonHook) GetPath() string { - return h.Path + return } -func (h *ModuleHook) handleModuleValuesPatch(currentValues utils.Values, valuesPatch utils.ValuesPatch) (*moduleValuesMergeResult, error) { - moduleValuesKey := utils.ModuleNameToValuesKey(h.Module.Name) - - if err := validateHookValuesPatch(valuesPatch, moduleValuesKey); err != nil { - return nil, fmt.Errorf("merge module '%s' values failed: %s", h.Module.Name, err) - } - newValuesRaw, valuesChanged, err := utils.ApplyValuesPatch(currentValues, valuesPatch) - if err != nil { - return nil, fmt.Errorf("merge module '%s' values failed: %s", h.Module.Name, err) - } - - result := &moduleValuesMergeResult{ - ModuleValuesKey: moduleValuesKey, - Values: utils.Values{moduleValuesKey: make(map[string]interface{})}, - ValuesChanged: valuesChanged, - ValuesPatch: valuesPatch, - } - - if moduleValuesRaw, hasKey := newValuesRaw[result.ModuleValuesKey]; hasKey { - moduleValues, ok := moduleValuesRaw.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("expected map at key '%s', got:\n%s", result.ModuleValuesKey, utils_data.YamlToString(moduleValuesRaw)) - } - result.Values[result.ModuleValuesKey] = moduleValues - result.ModuleValues = moduleValues - } - - return result, nil -} - -func validateHookValuesPatch(valuesPatch utils.ValuesPatch, acceptableKey string) error { - for _, op := range valuesPatch.Operations { - if op.Op == "replace" { - return fmt.Errorf("unsupported patch operation '%s': '%s'", op.Op, op.ToString()) +func LoadModuleHooksConfig(hooks []*ModuleHook) error { + for _, hook := range hooks { + configOutput, err := NewHookExecutor(hook, nil).Config() + if err != nil { + return fmt.Errorf("hook '%s' config: %s", hook.GetPath(), err) } - pathParts := strings.Split(op.Path, "/") - if len(pathParts) > 1 { - affectedKey := pathParts[1] - if affectedKey != acceptableKey { - return fmt.Errorf("unacceptable patch operation path '%s' (only '%s' accepted): '%s'", affectedKey, acceptableKey, op.ToString()) - } + err = hook.WithConfig(configOutput) + if err != nil { + return fmt.Errorf("creating global hook '%s': %s", hook.GetName(), err) } } return nil } -func (h *ModuleHook) run(bindingType BindingType, context []BindingContext) error { - moduleName := h.Module.Name - rlog.Infof("Running module hook '%s' binding '%s' ...", h.Name, bindingType) - - moduleHookExecutor := NewHookExecutor(h, context) - patches, err := moduleHookExecutor.Run() - if err != nil { - return fmt.Errorf("module hook '%s' failed: %s", h.Name, err) - } - - configValuesPatch, has := patches[utils.ConfigMapPatch] - if has && configValuesPatch != nil{ - preparedConfigValues := utils.MergeValues( - utils.Values{utils.ModuleNameToValuesKey(moduleName): map[string]interface{}{}}, - h.moduleManager.kubeModulesConfigValues[moduleName], - ) - - configValuesPatchResult, err := h.handleModuleValuesPatch(preparedConfigValues, *configValuesPatch) +func LoadGlobalHooksConfig(hooks []*GlobalHook) error { + for _, hook := range hooks { + configOutput, err := NewHookExecutor(hook, nil).Config() if err != nil { - return fmt.Errorf("module hook '%s': kube module config values update error: %s", h.Name, err) - } - if configValuesPatchResult.ValuesChanged { - err := h.moduleManager.kubeConfigManager.SetKubeModuleValues(moduleName, configValuesPatchResult.Values) - if err != nil { - rlog.Debugf("Module hook '%s' kube module config values stay unchanged:\n%s", utils.ValuesToString(h.moduleManager.kubeModulesConfigValues[moduleName])) - return fmt.Errorf("module hook '%s': set kube module config failed: %s", h.Name, err) - } - - h.moduleManager.kubeModulesConfigValues[moduleName] = configValuesPatchResult.Values - rlog.Debugf("Module hook '%s': kube module '%s' config values updated:\n%s", h.Name, moduleName, utils.ValuesToString(h.moduleManager.kubeModulesConfigValues[moduleName])) + return fmt.Errorf("hook '%s' config: %s", hook.GetPath(), err) } - } - valuesPatch, has := patches[utils.MemoryValuesPatch] - if has && valuesPatch != nil { - valuesPatchResult, err := h.handleModuleValuesPatch(h.values(), *valuesPatch) + err = hook.WithConfig(configOutput) if err != nil { - return fmt.Errorf("module hook '%s': dynamic module values update error: %s", h.Name, err) - } - if valuesPatchResult.ValuesChanged { - h.moduleManager.modulesDynamicValuesPatches[moduleName] = utils.AppendValuesPatch(h.moduleManager.modulesDynamicValuesPatches[moduleName], valuesPatchResult.ValuesPatch) - rlog.Debugf("Module hook '%s': dynamic module '%s' values updated:\n%s", h.Name, moduleName, utils.ValuesToString(h.values())) + return fmt.Errorf("creating module hook '%s': %s", hook.GetName(), err) } } return nil } -// PrepareTmpFilesForHookRun creates temporary files for hook and returns environment variables with paths -func (h *ModuleHook) PrepareTmpFilesForHookRun(context []BindingContext) (tmpFiles map[string]string, err error) { - tmpFiles = make(map[string]string, 0) - tmpFiles["CONFIG_VALUES_PATH"], err = h.prepareConfigValuesJsonFile() - if err != nil { - return - } - - tmpFiles["VALUES_PATH"], err = h.prepareValuesJsonFile() - if err != nil { - return - } - - if len(context) > 0 { - tmpFiles["BINDING_CONTEXT_PATH"], err = h.prepareBindingContextJsonFile(context) - if err != nil { - return - } - } +func (mm *MainModuleManager) RegisterGlobalHooks() error { + rlog.Debug("INIT: global hooks") - tmpFiles["CONFIG_VALUES_JSON_PATCH_PATH"], err= h.prepareConfigValuesJsonPatchFile() - if err != nil { - return - } + mm.globalHooksOrder = make(map[BindingType][]*GlobalHook) + mm.globalHooksByName = make(map[string]*GlobalHook) - tmpFiles["VALUES_JSON_PATCH_PATH"], err = h.prepareValuesJsonPatchFile() + hooks, err := SearchGlobalHooks(mm.GlobalHooksDir) if err != nil { - return + return err } - return -} - - -func (h *ModuleHook) configValues() utils.Values { - return h.Module.configValues() -} - -func (h *ModuleHook) values() utils.Values { - return h.Module.values() -} - -func (h *ModuleHook) prepareValuesJsonFile() (string, error) { - return h.Module.prepareValuesJsonFile() -} - -func (h *ModuleHook) prepareValuesYamlFile() (string, error) { - return h.Module.prepareValuesYamlFile() -} - -func (h *ModuleHook) prepareConfigValuesJsonFile() (string, error) { - return h.Module.prepareConfigValuesJsonFile() -} - -func (h *ModuleHook) prepareConfigValuesYamlFile() (string, error) { - return h.Module.prepareConfigValuesYamlFile() -} - -func (h *ModuleHook) prepareBindingContextJsonFile(context []BindingContext) (string, error) { - data, _ := json.Marshal(context) - //data := utils.MustDump(utils.DumpValuesJson(context)) - path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("%s.module-hook-%s-binding-context.json", h.Module.SafeName(), h.SafeName())) - err := dumpData(path, data) + err = LoadGlobalHooksConfig(hooks) if err != nil { - return "", err + return err } - rlog.Debugf("Prepared module %s hook %s binding context:\n%s", h.Module.SafeName(), h.Name, utils_data.YamlToString(context)) - - return path, nil -} - -func prepareHookConfig(hookConfig *HookConfig) { - for i := range hookConfig.OnKubernetesEvent { - config := &hookConfig.OnKubernetesEvent[i] - - if config.EventTypes == nil { - config.EventTypes = []kube_events_manager.OnKubernetesEventType{ - kube_events_manager.KubernetesEventOnAdd, - kube_events_manager.KubernetesEventOnUpdate, - kube_events_manager.KubernetesEventOnDelete, - } - } - - if config.NamespaceSelector == nil { - config.NamespaceSelector = &kube_events_manager.KubeNamespaceSelector{Any: true} + for _, globalHook := range hooks { + globalHook.WithModuleManager(mm) + // register global hook in indexes + for _, binding := range globalHook.Config.Bindings() { + mm.globalHooksOrder[binding] = append(mm.globalHooksOrder[binding], globalHook) } - } -} - -func (mm *MainModuleManager) initGlobalHooks() error { - rlog.Debug("INIT: global hooks") - - mm.globalHooksOrder = make(map[BindingType][]*GlobalHook) - mm.globalHooksByName = make(map[string]*GlobalHook) - - hooksDir := mm.GlobalHooksDir - - err := mm.initHooks(hooksDir, func(hookPath string, output []byte) error { - hookName, err := filepath.Rel(mm.GlobalHooksDir, hookPath) - if err != nil { - return err - } - - rlog.Infof("INIT: global hook '%s'", hookName) - - hookConfig := &GlobalHookConfig{} - if err := json.Unmarshal(output, hookConfig); err != nil { - return fmt.Errorf("INIT: cannot unmarshal config from global hook %s: %s\n%s", hookName, err.Error(), output) - } - - prepareHookConfig(&hookConfig.HookConfig) - - if err := mm.registerGlobalHook(hookName, hookPath, hookConfig); err != nil { - return fmt.Errorf("INIT: cannot add global hook '%s': %s", hookName, err.Error()) - } - - return nil - }) - - if err != nil { - return err + mm.globalHooksByName[globalHook.Name] = globalHook } return nil } -func (mm *MainModuleManager) InitModuleHooks(module *Module) error { +func (mm *MainModuleManager) RegisterModuleHooks(module *Module) error { if _, ok := mm.modulesHooksOrderByName[module.Name]; ok { rlog.Debugf("INIT: module '%s' hooks: already initialized", module.Name) return nil @@ -681,180 +187,26 @@ func (mm *MainModuleManager) InitModuleHooks(module *Module) error { rlog.Infof("INIT: module '%s' hooks ...", module.Name) - hooksDir := filepath.Join(module.Path, "hooks") - - err := mm.initHooks(hooksDir, func(hookPath string, output []byte) error { - hookName, err := filepath.Rel(filepath.Dir(module.Path), hookPath) - if err != nil { - return err - } - - rlog.Infof("INIT: hook '%s' ...", hookName) - - hookConfig := &ModuleHookConfig{} - if err := json.Unmarshal(output, hookConfig); err != nil { - return fmt.Errorf("unmarshaling module '%s' hook '%s' json failed: %s", module.SafeName(), hookName, err.Error()) - } - - prepareHookConfig(&hookConfig.HookConfig) - - if err := mm.registerModuleHook(module.Name, hookName, hookPath, hookConfig); err != nil { - return fmt.Errorf("adding module '%s' hook '%s' failed: %s", module.SafeName(), hookName, err.Error()) - } - - return nil - }) - + hooks, err := SearchModuleHooks(module) if err != nil { - // cleanup hook indexes on error - mm.removeModuleHooks(module.Name) return err } - return nil -} - -func (mm *MainModuleManager) initHooks(hooksDir string, addHookFn func(hookPath string, output []byte) error) error { - if _, err := os.Stat(hooksDir); os.IsNotExist(err) { - return nil - } - - // retrieve a list of executable files in hooksDir sorted by filename - hooksRelativePaths, _, err := utils.FindExecutableFilesInPath(hooksDir) + err = LoadModuleHooksConfig(hooks) if err != nil { return err } - for _, hookPath := range hooksRelativePaths { - cmd := makeCommand("", hookPath, []string{}, []string{"--config"}) - output, err := execCommandOutput(cmd) - if err != nil { - return fmt.Errorf("cannot get config for hook '%s': %s", hookPath, err) - } - - if err := addHookFn(hookPath, output); err != nil { - return err + for _, moduleHook := range hooks { + moduleHook.WithModuleManager(mm) + // register module hook in indexes + for _, binding := range moduleHook.Config.Bindings() { + if mm.modulesHooksOrderByName[module.Name] == nil { + mm.modulesHooksOrderByName[module.Name] = make(map[BindingType][]*ModuleHook) + } + mm.modulesHooksOrderByName[module.Name][binding] = append(mm.modulesHooksOrderByName[module.Name][binding], moduleHook) } } - return nil } - -func (h *GlobalHook) prepareConfigValuesJsonPatchFile() (string, error) { - path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("%s.global-hook-config-values.json-patch", h.SafeName())) - if err := createHookResultValuesFile(path); err != nil { - return "", err - } - return path, nil -} - -func (h *GlobalHook) prepareValuesJsonPatchFile() (string, error) { - path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("%s.global-hook-values.json-patch", h.SafeName())) - if err := createHookResultValuesFile(path); err != nil { - return "", err - } - return path, nil -} - -func (h *ModuleHook) prepareConfigValuesJsonPatchFile() (string, error) { - path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("%s.global-hook-config-values.json-patch", h.SafeName())) - if err := createHookResultValuesFile(path); err != nil { - return "", err - } - return path, nil -} - -func (h *ModuleHook) prepareValuesJsonPatchFile() (string, error) { - path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("%s.global-hook-values.json-patch", h.SafeName())) - if err := createHookResultValuesFile(path); err != nil { - return "", err - } - return path, nil -} - - -func createHookResultValuesFile(filePath string) error { - file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) - if err != nil { - return nil - } - - _ = file.Close() - return nil -} - -func makeCommand(dir string, entrypoint string, envs []string, args []string) *exec.Cmd { - envs = append(os.Environ(), envs...) - return executor.MakeCommand(dir, entrypoint, args, envs) -} - -func execCommandOutput(cmd *exec.Cmd) ([]byte, error) { - rlog.Debugf("Executing hook in %s: '%s'", cmd.Dir, strings.Join(cmd.Args, " ")) - cmd.Stdout = nil - - output, err := executor.Output(cmd) - if err != nil { - rlog.Errorf("Hook '%s' output:\n%s", strings.Join(cmd.Args, " "), string(output)) - return output, err - } - - rlog.Debugf("Hook '%s' output:\n%s", strings.Join(cmd.Args, " "), string(output)) - - return output, nil -} - - -type HookExecutor struct { - Hook Hook - Context []BindingContext - ConfigValuesPath string - ValuesPath string - ContextPath string - ConfigValuesPatchPath string - ValuesPatchPath string -} - -func NewHookExecutor(h Hook, context []BindingContext) *HookExecutor { - return &HookExecutor{ - Hook: h, - Context: context, - } -} - -func (e *HookExecutor) Run() (patches map[utils.ValuesPatchType]*utils.ValuesPatch, err error) { - patches = make(map[utils.ValuesPatchType]*utils.ValuesPatch) - - tmpFiles, err := e.Hook.PrepareTmpFilesForHookRun(e.Context) - if err != nil { - return nil, err - } - e.ConfigValuesPatchPath = tmpFiles["CONFIG_VALUES_JSON_PATCH_PATH"] - e.ValuesPatchPath = tmpFiles["VALUES_JSON_PATCH_PATH"] - - envs := []string{} - envs = append(envs, os.Environ()...) - for envName, filePath := range tmpFiles { - envs = append(envs, fmt.Sprintf("%s=%s", envName, filePath)) - } - envs = append(envs, helm.Client.CommandEnv()...) - - cmd := executor.MakeCommand("", e.Hook.GetPath(), []string{}, envs) - - err = executor.Run(cmd, true) - if err != nil { - return nil, fmt.Errorf("%s FAILED: %s", e.Hook.GetName(), err) - } - - patches[utils.ConfigMapPatch], err = utils.ValuesPatchFromFile(e.ConfigValuesPatchPath) - if err != nil { - return nil, fmt.Errorf("got bad config values json patch from hook %s: %s", e.Hook.GetName(), err) - } - - patches[utils.MemoryValuesPatch], err = utils.ValuesPatchFromFile(e.ValuesPatchPath) - if err != nil { - return nil, fmt.Errorf("got bad values json patch from hook %s: %s", e.Hook.GetName(), err) - } - - return patches, nil -} diff --git a/pkg/module_manager/hook/kube_event/hooks_controller.go b/pkg/module_manager/hook/kube_event/hooks_controller.go index 83546509..32631931 100644 --- a/pkg/module_manager/hook/kube_event/hooks_controller.go +++ b/pkg/module_manager/hook/kube_event/hooks_controller.go @@ -3,125 +3,140 @@ package kube_event import ( "fmt" - "github.com/flant/addon-operator/pkg/module_manager" - "github.com/flant/addon-operator/pkg/task" + "github.com/flant/shell-operator/pkg/hook" "github.com/flant/shell-operator/pkg/hook/kube_event" "github.com/flant/shell-operator/pkg/kube_events_manager" - "github.com/romana/rlog" + "github.com/flant/addon-operator/pkg/module_manager" + "github.com/flant/addon-operator/pkg/task" ) -// MakeKubeEventHookDescriptors converts hook config into KubeEventHook structures -func MakeKubeEventHookDescriptors(hook module_manager.Hook, hookConfig *module_manager.HookConfig) []*kube_event.KubeEventHook { - res := make([]*kube_event.KubeEventHook, 0) - - for _, config := range hookConfig.OnKubernetesEvent { - if config.NamespaceSelector.Any { - res = append(res, ConvertOnKubernetesEventToKubeEventHook(hook, config, "")) - } else { - for _, namespace := range config.NamespaceSelector.MatchNames { - res = append(res, ConvertOnKubernetesEventToKubeEventHook(hook, config, namespace)) - } - } - } - - return res +type KubernetesHooksController interface { + WithModuleManager(moduleManager module_manager.ModuleManager) + WithKubeEventsManager(kube_events_manager.KubeEventsManager) + EnableGlobalHooks() error + EnableModuleHooks(moduleName string) error + DisableModuleHooks(moduleName string) error + HandleEvent(kubeEvent kube_events_manager.KubeEvent) ([]task.Task, error) } -func ConvertOnKubernetesEventToKubeEventHook(hook module_manager.Hook, config kube_events_manager.OnKubernetesEventConfig, namespace string) *kube_event.KubeEventHook { - return &kube_event.KubeEventHook{ - HookName: hook.GetName(), - Name: config.Name, - EventTypes: config.EventTypes, - Kind: config.Kind, - Namespace: namespace, - Selector: config.Selector, - JqFilter: config.JqFilter, - AllowFailure: config.AllowFailure, - Debug: !config.DisableDebug, - } -} - -type KubeEventsHooksController interface { - EnableGlobalHooks(moduleManager module_manager.ModuleManager, eventsManager kube_events_manager.KubeEventsManager) error - EnableModuleHooks(moduleName string, moduleManager module_manager.ModuleManager, eventsManager kube_events_manager.KubeEventsManager) error - DisableModuleHooks(moduleName string, moduleManager module_manager.ModuleManager, eventsManager kube_events_manager.KubeEventsManager) error - HandleEvent(kubeEvent kube_events_manager.KubeEvent) (*struct{ Tasks []task.Task }, error) -} - -type MainKubeEventsHooksController struct { +type kubernetesHooksController struct { + // storages for registered module hooks and global hooks GlobalHooks map[string]*kube_event.KubeEventHook ModuleHooks map[string]*kube_event.KubeEventHook EnabledModules []string + + // dependencies + moduleManager module_manager.ModuleManager + kubeEventsManager kube_events_manager.KubeEventsManager } -// NewMainKubeEventsHooksController returns new instance of MainKubeEventsHooksController -func NewMainKubeEventsHooksController() *MainKubeEventsHooksController { - obj := &MainKubeEventsHooksController{} +// kubernetesHooksController should implement KubernetesHooksController +var _ KubernetesHooksController = &kubernetesHooksController{} + +// NewKubernetesHooksController returns new instance of kubernetesHooksController +func NewKubernetesHooksController() *kubernetesHooksController { + obj := &kubernetesHooksController{} obj.GlobalHooks = make(map[string]*kube_event.KubeEventHook) obj.ModuleHooks = make(map[string]*kube_event.KubeEventHook) obj.EnabledModules = make([]string, 0) return obj } +func (c *kubernetesHooksController) WithModuleManager(mm module_manager.ModuleManager) { + c.moduleManager = mm +} + +func (c *kubernetesHooksController) WithKubeEventsManager(kem kube_events_manager.KubeEventsManager) { + c.kubeEventsManager = kem +} + // EnableGlobalHooks starts kube events informers for all global hooks -func (obj *MainKubeEventsHooksController) EnableGlobalHooks(moduleManager module_manager.ModuleManager, eventsManager kube_events_manager.KubeEventsManager) error { - globalHooks := moduleManager.GetGlobalHooksInOrder(module_manager.KubeEvents) +func (c *kubernetesHooksController) EnableGlobalHooks() error { + globalHooks := c.moduleManager.GetGlobalHooksInOrder(module_manager.KubeEvents) for _, globalHookName := range globalHooks { - globalHook, _ := moduleManager.GetGlobalHook(globalHookName) + globalHook, _ := c.moduleManager.GetGlobalHook(globalHookName) - for _, desc := range MakeKubeEventHookDescriptors(globalHook, &globalHook.Config.HookConfig) { - configId, err := eventsManager.Run(desc.EventTypes, desc.Kind, desc.Namespace, desc.Selector, desc.ObjectName, desc.JqFilter, desc.Debug) + for _, config := range globalHook.Config.OnKubernetesEvents { + err := c.kubeEventsManager.AddMonitor("", config.Monitor) if err != nil { - return err + return fmt.Errorf("run kube monitor for hook %s: %s", globalHook.Name, err) + } + c.GlobalHooks[config.Monitor.Metadata.ConfigId] = &kube_event.KubeEventHook{ + HookName: globalHook.Name, + ConfigName: config.ConfigName, + AllowFailure: config.AllowFailure, } - obj.GlobalHooks[configId] = desc - - rlog.Debugf("MAIN: run informer %s for global hook %s", configId, globalHook.Name) } + // + //for _, desc := range MakeKubeEventHookDescriptors(globalHook, &globalHook.Config.HookConfig) { + // configId, err := c.kubeEventsManager.Run(desc.EventTypes, desc.Kind, desc.Namespace, desc.Selector, desc.ObjectName, desc.JqFilter, desc.Debug) + // if err != nil { + // return err + // } + // c.GlobalHooks[configId] = desc + // + // rlog.Debugf("MAIN: run informer %s for global hook %s", configId, globalHook.Name) + //} } return nil } // EnableModuleHooks starts kube events informers for all module hooks -func (obj *MainKubeEventsHooksController) EnableModuleHooks(moduleName string, moduleManager module_manager.ModuleManager, eventsManager kube_events_manager.KubeEventsManager) error { - for _, enabledModuleName := range obj.EnabledModules { +func (c *kubernetesHooksController) EnableModuleHooks(moduleName string) error { + for _, enabledModuleName := range c.EnabledModules { if enabledModuleName == moduleName { // already enabled return nil } } - moduleHooks, err := moduleManager.GetModuleHooksInOrder(moduleName, module_manager.KubeEvents) + moduleHooks, err := c.moduleManager.GetModuleHooksInOrder(moduleName, module_manager.KubeEvents) if err != nil { return err } for _, moduleHookName := range moduleHooks { - moduleHook, _ := moduleManager.GetModuleHook(moduleHookName) + moduleHook, _ := c.moduleManager.GetModuleHook(moduleHookName) - for _, desc := range MakeKubeEventHookDescriptors(moduleHook, &moduleHook.Config.HookConfig) { - configId, err := eventsManager.Run(desc.EventTypes, desc.Kind, desc.Namespace, desc.Selector, desc.ObjectName, desc.JqFilter, desc.Debug) + for _, config := range moduleHook.Config.OnKubernetesEvents { + err := c.kubeEventsManager.AddMonitor("", config.Monitor) if err != nil { - return err + return fmt.Errorf("run kube monitor for hook %s: %s", moduleHook.Name, err) + } + c.ModuleHooks[config.Monitor.Metadata.ConfigId] = &kube_event.KubeEventHook{ + HookName: moduleHook.Name, + ConfigName: config.ConfigName, + AllowFailure: config.AllowFailure, } - obj.ModuleHooks[configId] = desc - - rlog.Debugf("MAIN: run informer %s for module hook %s", configId, moduleHook.Name) } + + //for _, desc := range MakeKubeEventHookDescriptors(moduleHook, &moduleHook.Config.HookConfig) { + // configId, err := c.kubeEventsManager.Run(desc.EventTypes, desc.Kind, desc.Namespace, desc.Selector, desc.ObjectName, desc.JqFilter, desc.Debug) + // if err != nil { + // return err + // } + // c.ModuleHooks[configId] = desc + // + // rlog.Debugf("MAIN: run informer %s for module hook %s", configId, moduleHook.Name) + //} } - obj.EnabledModules = append(obj.EnabledModules, moduleName) + c.EnabledModules = append(c.EnabledModules, moduleName) + // Start informers for new monitors + c.kubeEventsManager.Start() return nil } -// DisableModuleHooks stops informers for module hooks -func (obj *MainKubeEventsHooksController) DisableModuleHooks(moduleName string, moduleManager module_manager.ModuleManager, eventsManager kube_events_manager.KubeEventsManager) error { +// DisableModuleHooks stops all monitors for all hooks in module +func (c *kubernetesHooksController) DisableModuleHooks(moduleName string) error { + // TODO remove EnabledModules index. ConfigId is now in moduleHook.Config.OnKubernetesEvents[].Monitor.Metadata.ConfigId + // loop through module hooks and check if configId is in c.ModuleHooks, stop monitor and delete a map item. moduleEnabledInd := -1 - for i, enabledModuleName := range obj.EnabledModules { + for i, enabledModuleName := range c.EnabledModules { if enabledModuleName == moduleName { moduleEnabledInd = i break @@ -130,22 +145,23 @@ func (obj *MainKubeEventsHooksController) DisableModuleHooks(moduleName string, if moduleEnabledInd < 0 { return nil } - obj.EnabledModules = append(obj.EnabledModules[:moduleEnabledInd], obj.EnabledModules[moduleEnabledInd+1:]...) + // remove name from enabled modules index + c.EnabledModules = append(c.EnabledModules[:moduleEnabledInd], c.EnabledModules[moduleEnabledInd+1:]...) - disabledModuleHooks, err := moduleManager.GetModuleHooksInOrder(moduleName, module_manager.KubeEvents) + disabledModuleHooks, err := c.moduleManager.GetModuleHooksInOrder(moduleName, module_manager.KubeEvents) if err != nil { return err } - for configId, desc := range obj.ModuleHooks { + for configId, desc := range c.ModuleHooks { for _, disabledModuleHookName := range disabledModuleHooks { if desc.HookName == disabledModuleHookName { - err := eventsManager.Stop(configId) + err := c.kubeEventsManager.StopMonitor(configId) if err != nil { return err } - delete(obj.ModuleHooks, configId) + delete(c.ModuleHooks, configId) break } @@ -156,45 +172,112 @@ func (obj *MainKubeEventsHooksController) DisableModuleHooks(moduleName string, } // HandleEvent creates a task from kube event -func (obj *MainKubeEventsHooksController) HandleEvent(kubeEvent kube_events_manager.KubeEvent) (*struct{ Tasks []task.Task }, error) { - res := &struct{ Tasks []task.Task }{Tasks: make([]task.Task, 0)} - var desc *kube_event.KubeEventHook - var taskType task.TaskType +func (c *kubernetesHooksController) HandleEvent(kubeEvent kube_events_manager.KubeEvent) ([]task.Task, error) { + res := make([]task.Task, 0) - if moduleDesc, hasKey := obj.ModuleHooks[kubeEvent.ConfigId]; hasKey { - desc = moduleDesc - taskType = task.ModuleHookRun - } else if globalDesc, hasKey := obj.GlobalHooks[kubeEvent.ConfigId]; hasKey { - desc = globalDesc + globalHook, hasGlobalHook := c.GlobalHooks[kubeEvent.ConfigId] + moduleHook, hasModuleHook := c.ModuleHooks[kubeEvent.ConfigId] + if !hasGlobalHook && !hasModuleHook { + return nil, fmt.Errorf("Possible a bug: kubernets event '%s/%s/%s %s' is received, but no hook is found", kubeEvent.Namespace, kubeEvent.Kind, kubeEvent.Name, kubeEvent.Event) + } + + var taskType task.TaskType + var kubeHook *kube_event.KubeEventHook + if hasGlobalHook { + kubeHook = globalHook taskType = task.GlobalHookRun + } else { + kubeHook = moduleHook + taskType = task.ModuleHookRun } - if desc != nil && taskType != "" { - bindingName := desc.Name - if desc.Name == "" { - bindingName = module_manager.ContextBindingType[module_manager.KubeEvents] + switch kubeEvent.Type { + case "Synchronization": + // Send all objects + objList := make([]interface{}, 0) + for _, obj := range kubeEvent.Objects { + objList = append(objList, interface{}(obj)) } + bindingContext := make([]module_manager.BindingContext, 0) + bindingContext = append(bindingContext, module_manager.BindingContext{ + BindingContext: hook.BindingContext{ + Binding: kubeHook.ConfigName, + Type: kubeEvent.Type, + Objects: objList, + }, + }) + + newTask := task.NewTask(taskType, kubeHook.HookName). + WithBinding(module_manager.KubeEvents). + WithBindingContext(bindingContext). + WithAllowFailure(kubeHook.AllowFailure) + res = append(res, newTask) + default: bindingContext := make([]module_manager.BindingContext, 0) - for _, kEvent := range kubeEvent.Events { + for _, kEvent := range kubeEvent.WatchEvents { bindingContext = append(bindingContext, module_manager.BindingContext{ - Binding: bindingName, - ResourceEvent: kEvent, - ResourceNamespace: kubeEvent.Namespace, - ResourceKind: kubeEvent.Kind, - ResourceName: kubeEvent.Name, + BindingContext: hook.BindingContext{ + Binding: kubeHook.ConfigName, + Type: "Event", + WatchEvent: kEvent, + + Namespace: kubeEvent.Namespace, + Kind: kubeEvent.Kind, + Name: kubeEvent.Name, + + Object: kubeEvent.Object, + FilterResult: kubeEvent.FilterResult, + }, }) } - newTask := task.NewTask(taskType, desc.HookName). + newTask := task.NewTask(taskType, kubeHook.HookName). WithBinding(module_manager.KubeEvents). WithBindingContext(bindingContext). - WithAllowFailure(desc.Config.AllowFailure) + WithAllowFailure(kubeHook.AllowFailure) - res.Tasks = append(res.Tasks, newTask) - } else { - return nil, fmt.Errorf("Unknown kube event: no such config id '%s' registered", kubeEvent.ConfigId) + res = append(res, newTask) } + + //var desc *kube_event.KubeEventHook + //var taskType task.TaskType + + //if moduleDesc, hasKey := c.ModuleHooks[kubeEvent.ConfigId]; hasKey { + // desc = moduleDesc + // taskType = task.ModuleHookRun + //} else if globalDesc, hasKey := c.GlobalHooks[kubeEvent.ConfigId]; hasKey { + // desc = globalDesc + // taskType = task.GlobalHookRun + //} + + //if desc != nil && taskType != "" { + // bindingName := desc.Name + // if desc.Name == "" { + // bindingName = module_manager.ContextBindingType[module_manager.KubeEvents] + // } + // + // bindingContext := make([]module_manager.BindingContext, 0) + // for _, kEvent := range kubeEvent.Events { + // bindingContext = append(bindingContext, module_manager.BindingContext{ + // Binding: bindingName, + // ResourceEvent: kEvent, + // ResourceNamespace: kubeEvent.Namespace, + // ResourceKind: kubeEvent.Kind, + // ResourceName: kubeEvent.Name, + // }) + // } + // + // newTask := task.NewTask(taskType, desc.HookName). + // WithBinding(module_manager.KubeEvents). + // WithBindingContext(bindingContext). + // WithAllowFailure(desc.Config.AllowFailure) + // + // res = append(res, newTask) + //} else { + // return nil, fmt.Errorf("Unknown kube event: no such config id '%s' registered", kubeEvent.ConfigId) + //} + return res, nil } diff --git a/pkg/module_manager/hook/schedule/hook_controller_test.go b/pkg/module_manager/hook/schedule/hook_controller_test.go new file mode 100644 index 00000000..19c4c977 --- /dev/null +++ b/pkg/module_manager/hook/schedule/hook_controller_test.go @@ -0,0 +1,227 @@ +package schedule + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + hook2 "github.com/flant/shell-operator/pkg/hook" + + "github.com/flant/addon-operator/pkg/module_manager" +) + + +type ScheduleManagerMock struct { + Added map[string]bool + Removed map[string]bool +} + +func NewScheduleManagerMock() *ScheduleManagerMock { + return &ScheduleManagerMock{ + Added: make(map[string]bool, 0), + Removed: make(map[string]bool, 0), + } +} + +func (s *ScheduleManagerMock) Add(crontab string) (string, error) { + s.Added[crontab] = true + return crontab, nil +} + +func (s *ScheduleManagerMock) Remove(entryId string) error { + s.Removed[entryId] = true + return nil +} + +func (s *ScheduleManagerMock) Run() { + return +} + +func Test_Schedule_Controller_UpdateHooks(t *testing.T) { + var sm *ScheduleManagerMock + + var globalHook1 = &module_manager.GlobalHook{ + CommonHook: &module_manager.CommonHook{ + Name: "hook-global", + }, + Config: &module_manager.GlobalHookConfig{ + HookConfig: hook2.HookConfig{ + Schedules: []hook2.ScheduleConfig{ + { + Crontab: "*/5 * * * * *", + }, + }, + }, + }, + } + + var moduleHook1 = &module_manager.ModuleHook{ + CommonHook: &module_manager.CommonHook{ + Name: "hook-module-1", + }, + Config: &module_manager.ModuleHookConfig{ + HookConfig: hook2.HookConfig{ + Schedules: []hook2.ScheduleConfig{ + { + Crontab: "*/10 * * * * *", + }, + }, + }, + }, + } + + var moduleHook2 = &module_manager.ModuleHook{ + CommonHook: &module_manager.CommonHook{ + Name: "hook-module-2", + }, + Config: &module_manager.ModuleHookConfig{ + HookConfig: hook2.HookConfig{ + Schedules: []hook2.ScheduleConfig{ + { + Crontab: "*/20 * * * * *", + }, + }, + }, + }, + } + + var defaultGlobalHooks = map[string]*module_manager.GlobalHook{ + "hook-global": globalHook1, + } + var defaultModuleHooks = map[string]map[string]*module_manager.ModuleHook{} + + var globalHooksMock map[string]*module_manager.GlobalHook + var moduleHooksMock map[string]map[string]*module_manager.ModuleHook + var mm = module_manager.NewModuleManagerMock(module_manager.ModuleManagerMockFns{ + GetGlobalHook: func(name string) (hook *module_manager.GlobalHook, e error) { + return globalHooksMock[name], nil + }, + GetModuleHook: func(name string) (hook *module_manager.ModuleHook, e error) { + for _, moduleHooks := range moduleHooksMock{ + if hook, ok := moduleHooks[name]; ok { + return hook, nil + } + } + return nil, nil + }, + GetGlobalHooksInOrder: func(bindingType module_manager.BindingType) []string { + res := []string{} + for k := range globalHooksMock{ + res = append(res, k) + } + return res + }, + GetModuleNamesInOrder: func() []string { + res := []string{} + for k := range moduleHooksMock{ + res = append(res, k) + } + return res + }, + GetModuleHooksInOrder: func(moduleName string, bindingType module_manager.BindingType) (strings []string, e error) { + res := []string{} + for k := range moduleHooksMock[moduleName]{ + res = append(res, k) + } + return res, nil + }, + }) + + + var err error + var ctrl *scheduleHooksController + tests := []struct{ + name string + assertion func() + }{ + { + "update with global hook", + func() { + assert.NoError(t, err) + assert.Equal(t, true, sm.Added["*/5 * * * * *"]) + assert.Len(t, ctrl.GlobalHooks, 1) + assert.Len(t, ctrl.GlobalHooks.GetCrontabs(), 1) + assert.Len(t, ctrl.GlobalHooks.GetHooksForSchedule("*/5 * * * * *"), 1) + hooks := ctrl.GlobalHooks.GetHooksForSchedule("*/5 * * * * *") + assert.Equal(t, "hook-global", hooks[0].HookName) + }, + }, + { + "module enable", + func() { + moduleHooksMock["module-1"] = map[string]*module_manager.ModuleHook{ + "hook-module-1": moduleHook1, + } + ctrl.UpdateScheduleHooks() + + assert.Equal(t, true, sm.Added["*/10 * * * * *"]) + assert.Len(t, ctrl.ModuleHooks, 1) + assert.Len(t, ctrl.ModuleHooks.GetCrontabs(), 1) + assert.Len(t, ctrl.ModuleHooks.GetHooksForSchedule("*/10 * * * * *"), 1) + hooks := ctrl.ModuleHooks.GetHooksForSchedule("*/10 * * * * *") + assert.Equal(t, "hook-module-1", hooks[0].HookName) + }, + }, + { + "enable two modules", + func() { + moduleHooksMock["module-1"] = map[string]*module_manager.ModuleHook{ + "hook-module-1": moduleHook1, + } + moduleHooksMock["module-2"] = map[string]*module_manager.ModuleHook{ + "hook-module-2": moduleHook2, + } + ctrl.UpdateScheduleHooks() + + assert.Equal(t, true, sm.Added["*/10 * * * * *"]) + assert.Equal(t, true, sm.Added["*/20 * * * * *"]) + assert.Len(t, ctrl.ModuleHooks, 2) + assert.Len(t, ctrl.ModuleHooks.GetCrontabs(), 2) + assert.Len(t, ctrl.ModuleHooks.GetHooksForSchedule("*/20 * * * * *"), 1) + hooks := ctrl.ModuleHooks.GetHooksForSchedule("*/20 * * * * *") + assert.Equal(t, "hook-module-2", hooks[0].HookName) + }, + }, + { + "disable module", + func() { + moduleHooksMock["module-1"] = map[string]*module_manager.ModuleHook{ + "hook-module-1": moduleHook1, + } + moduleHooksMock["module-2"] = map[string]*module_manager.ModuleHook{ + "hook-module-2": moduleHook2, + } + ctrl.UpdateScheduleHooks() + + delete(moduleHooksMock, "module-2") + ctrl.UpdateScheduleHooks() + + assert.Equal(t, true, sm.Removed["*/20 * * * * *"]) + assert.Len(t, ctrl.ModuleHooks, 1) + assert.Len(t, ctrl.ModuleHooks.GetCrontabs(), 1) + assert.Len(t, ctrl.ModuleHooks.GetHooksForSchedule("*/10 * * * * *"), 1) + assert.Len(t, ctrl.ModuleHooks.GetHooksForSchedule("*/20 * * * * *"), 0) + hooks := ctrl.ModuleHooks.GetHooksForSchedule("*/10 * * * * *") + assert.Equal(t, "hook-module-1", hooks[0].HookName) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + sm = NewScheduleManagerMock() + + globalHooksMock = defaultGlobalHooks + moduleHooksMock = defaultModuleHooks + + ctrl = &scheduleHooksController{} + ctrl.WithModuleManager(mm) + ctrl.WithScheduleManager(sm) + + ctrl.UpdateScheduleHooks() + + test.assertion() + }) + } +} + diff --git a/pkg/module_manager/hook/schedule/hooks_controller.go b/pkg/module_manager/hook/schedule/hooks_controller.go new file mode 100644 index 00000000..ef0d2e37 --- /dev/null +++ b/pkg/module_manager/hook/schedule/hooks_controller.go @@ -0,0 +1,156 @@ +package schedule + +import ( + "fmt" + + "github.com/flant/shell-operator/pkg/hook" + "github.com/romana/rlog" + + "github.com/flant/shell-operator/pkg/hook/schedule" + "github.com/flant/shell-operator/pkg/schedule_manager" + + "github.com/flant/addon-operator/pkg/module_manager" + "github.com/flant/addon-operator/pkg/task" +) + +type ScheduleHooksController interface { + WithModuleManager(moduleManager module_manager.ModuleManager) + WithScheduleManager(schedule_manager.ScheduleManager) + + UpdateScheduleHooks() + + HandleEvent(crontab string) ([]task.Task, error) +} + +type scheduleHooksController struct { + GlobalHooks schedule.ScheduleHooksStorage + ModuleHooks schedule.ScheduleHooksStorage + + // dependencies + moduleManager module_manager.ModuleManager + scheduleManager schedule_manager.ScheduleManager +} + +var NewScheduleHooksController = func() ScheduleHooksController { + return &scheduleHooksController{ + GlobalHooks: make(schedule.ScheduleHooksStorage, 0), + ModuleHooks: make(schedule.ScheduleHooksStorage, 0), + } +} + +func (c *scheduleHooksController) WithModuleManager(moduleManager module_manager.ModuleManager) { + c.moduleManager = moduleManager +} + +func (c *scheduleHooksController) WithScheduleManager(scheduleManager schedule_manager.ScheduleManager) { + c.scheduleManager = scheduleManager +} + +// UpdateScheduledHooks recreates a new Hooks array. Note that many hooks can be bind +// to one crontab. +func (c *scheduleHooksController) UpdateScheduleHooks() { + // Load active crontabs for global hooks + newGlobalHooks := make(schedule.ScheduleHooksStorage, 0) + globalHooks := c.moduleManager.GetGlobalHooksInOrder(module_manager.Schedule) + + for _, hookName := range globalHooks { + globalHook, _ := c.moduleManager.GetGlobalHook(hookName) + for _, scheduleCfg := range globalHook.Config.Schedules { + _, err := c.scheduleManager.Add(scheduleCfg.Crontab) + if err != nil { + rlog.Errorf("Schedule: cannot add '%s' for hook '%s': %s", scheduleCfg.Crontab, hookName, err) + continue + } + rlog.Debugf("Schedule: add '%s' for hook '%s'", scheduleCfg.Crontab, hookName) + newGlobalHooks.AddHook(globalHook.Name, scheduleCfg) + } + } + + // Load active crontabs for module hooks + newModuleHooks := make(schedule.ScheduleHooksStorage, 0) + modules := c.moduleManager.GetModuleNamesInOrder() + + for _, moduleName := range modules { + moduleHooks, _ := c.moduleManager.GetModuleHooksInOrder(moduleName, module_manager.Schedule) + for _, moduleHookName := range moduleHooks { + moduleHook, _ := c.moduleManager.GetModuleHook(moduleHookName) + for _, scheduleCfg := range moduleHook.Config.Schedules { + _, err := c.scheduleManager.Add(scheduleCfg.Crontab) + if err != nil { + rlog.Errorf("Schedule: cannot add '%s' for hook '%s': %s", scheduleCfg.Crontab, moduleHookName, err) + continue + } + rlog.Debugf("Schedule: add '%s' for hook '%s'", scheduleCfg.Crontab, moduleHookName) + newModuleHooks.AddHook(moduleHook.Name, scheduleCfg) + } + } + } + + // Calculate obsolete crontab strings + oldCrontabs := map[string]bool{} + for _, crontab := range c.GlobalHooks.GetCrontabs() { + oldCrontabs[crontab] = false + } + for _, crontab := range c.ModuleHooks.GetCrontabs() { + oldCrontabs[crontab] = false + } + for _, crontab := range newGlobalHooks.GetCrontabs() { + oldCrontabs[crontab] = true + } + for _, crontab := range newModuleHooks.GetCrontabs() { + oldCrontabs[crontab] = true + } + + // Stop crontabs that is not in new storages. + for crontab, isActive := range oldCrontabs { + if !isActive { + c.scheduleManager.Remove(crontab) + } + } + + c.GlobalHooks = newGlobalHooks + c.ModuleHooks = newModuleHooks +} + +func (c *scheduleHooksController) HandleEvent(crontab string) ([]task.Task, error) { + res := make([]task.Task, 0) + + // search for global hooks by crontab + scheduleGlobalHooks := c.GlobalHooks.GetHooksForSchedule(crontab) + + for _, scheduleHook := range scheduleGlobalHooks { + _, err := c.moduleManager.GetGlobalHook(scheduleHook.HookName) + if err != nil { + rlog.Errorf("Possible a bug: global hook '%s' is registered for schedule but not found", scheduleHook.HookName) + continue + } + newTask := task.NewTask(task.GlobalHookRun, scheduleHook.HookName). + WithBinding(module_manager.Schedule). + AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: scheduleHook.ConfigName}}). + WithAllowFailure(scheduleHook.AllowFailure) + res = append(res, newTask) + } + + // search for module hooks by crontab + scheduleModuleHooks := c.ModuleHooks.GetHooksForSchedule(crontab) + + for _, scheduleHook := range scheduleModuleHooks { + _, err := c.moduleManager.GetModuleHook(scheduleHook.HookName) + if err != nil { + rlog.Errorf("Possible a bug: module hook '%s' is registered for schedule but not found", scheduleHook.HookName) + continue + } + newTask := task.NewTask(task.ModuleHookRun, scheduleHook.HookName). + WithBinding(module_manager.Schedule). + AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: scheduleHook.ConfigName}}). + WithAllowFailure(scheduleHook.AllowFailure) + res = append(res, newTask) + } + + if len(res) == 0 { + // NOTE should not happen + return nil, fmt.Errorf("Possible a bug: schedule event for crontab '%s' received, but no hook is found", crontab) + } + + return res, nil +} diff --git a/pkg/module_manager/hook_executor.go b/pkg/module_manager/hook_executor.go new file mode 100644 index 00000000..cba46f06 --- /dev/null +++ b/pkg/module_manager/hook_executor.go @@ -0,0 +1,90 @@ +package module_manager + +import ( + "fmt" + "os" + "strings" + + "github.com/romana/rlog" + + "github.com/flant/shell-operator/pkg/executor" + + "github.com/flant/addon-operator/pkg/helm" + "github.com/flant/addon-operator/pkg/utils" +) + +type HookExecutor struct { + Hook Hook + Context interface{} + ConfigValuesPath string + ValuesPath string + ContextPath string + ConfigValuesPatchPath string + ValuesPatchPath string +} + +func NewHookExecutor(h Hook, context interface{}) *HookExecutor { + return &HookExecutor{ + Hook: h, + Context: context, + } +} + +func (e *HookExecutor) Run() (patches map[utils.ValuesPatchType]*utils.ValuesPatch, err error) { + patches = make(map[utils.ValuesPatchType]*utils.ValuesPatch) + + tmpFiles, err := e.Hook.PrepareTmpFilesForHookRun(e.Context) + if err != nil { + return nil, err + } + e.ConfigValuesPatchPath = tmpFiles["CONFIG_VALUES_JSON_PATCH_PATH"] + e.ValuesPatchPath = tmpFiles["VALUES_JSON_PATCH_PATH"] + + envs := []string{} + envs = append(envs, os.Environ()...) + for envName, filePath := range tmpFiles { + envs = append(envs, fmt.Sprintf("%s=%s", envName, filePath)) + } + envs = append(envs, helm.Client.CommandEnv()...) + + cmd := executor.MakeCommand("", e.Hook.GetPath(), []string{}, envs) + + err = executor.Run(cmd) + if err != nil { + return nil, fmt.Errorf("%s FAILED: %s", e.Hook.GetName(), err) + } + + patches[utils.ConfigMapPatch], err = utils.ValuesPatchFromFile(e.ConfigValuesPatchPath) + if err != nil { + return nil, fmt.Errorf("got bad config values json patch from hook %s: %s", e.Hook.GetName(), err) + } + + patches[utils.MemoryValuesPatch], err = utils.ValuesPatchFromFile(e.ValuesPatchPath) + if err != nil { + return nil, fmt.Errorf("got bad values json patch from hook %s: %s", e.Hook.GetName(), err) + } + + return patches, nil +} + +func (e *HookExecutor) Config() (configOutput []byte, err error) { + envs := []string{} + envs = append(envs, os.Environ()...) + envs = append(envs, helm.Client.CommandEnv()...) + + cmd := executor.MakeCommand("", e.Hook.GetPath(), []string{"--config"}, envs) + + rlog.Debugf("Executing hook in %s: '%s'", cmd.Dir, strings.Join(cmd.Args, " ")) + cmd.Stdout = nil + + output, err := executor.Output(cmd) + if err != nil { + rlog.Errorf("Hook '%s' config failed: %v, output:\n%s", e.Hook.GetName(), err, string(output)) + return nil, fmt.Errorf("%s FAILED: %s", e.Hook.GetName(), err) + } + + rlog.Debugf("Hook '%s' config output:\n%s", e.Hook.GetName(), string(output)) + + return output, nil +} + diff --git a/pkg/module_manager/module.go b/pkg/module_manager/module.go index e84609d0..9cbed3bd 100644 --- a/pkg/module_manager/module.go +++ b/pkg/module_manager/module.go @@ -14,6 +14,7 @@ import ( "gopkg.in/yaml.v2" "github.com/flant/shell-operator/pkg/executor" + hook2 "github.com/flant/shell-operator/pkg/hook" utils_file "github.com/flant/shell-operator/pkg/utils/file" "github.com/flant/addon-operator/pkg/app" @@ -227,7 +228,14 @@ func (m *Module) runHooksByBinding(binding BindingType) error { return err } - if err := moduleHook.run(binding, []BindingContext{{Binding: ContextBindingType[binding]}}); err != nil { + err = moduleHook.run(binding, []BindingContext{ + { + BindingContext: hook2.BindingContext{ + Binding: ContextBindingType[binding], + }, + }, + }) + if err != nil { return err } } @@ -397,7 +405,7 @@ func (m *Module) moduleValuesKey() string { func (m *Module) prepareModuleEnabledResultFile() (string, error) { path := filepath.Join(m.moduleManager.TempDir, fmt.Sprintf("%s.module-enabled-result", m.Name)) - if err := createHookResultValuesFile(path); err != nil { + if err := CreateEmptyWritableFile(path); err != nil { return "", err } return path, nil @@ -460,7 +468,7 @@ func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string) (bool, cmd := executor.MakeCommand("", enabledScriptPath, []string{}, envs) - if err := executor.Run(cmd, true); err != nil { + if err := executor.Run(cmd); err != nil { return false, err } diff --git a/pkg/module_manager/module_hook.go b/pkg/module_manager/module_hook.go new file mode 100644 index 00000000..3882ab41 --- /dev/null +++ b/pkg/module_manager/module_hook.go @@ -0,0 +1,242 @@ +package module_manager + +import ( + "encoding/json" + "fmt" + "path/filepath" + + "github.com/romana/rlog" + + hook2 "github.com/flant/shell-operator/pkg/hook" + utils_data "github.com/flant/shell-operator/pkg/utils/data" + + "github.com/flant/addon-operator/pkg/utils" +) + +type ModuleHook struct { + *CommonHook + Module *Module + Config *ModuleHookConfig +} +var _ Hook = &ModuleHook{} + +func NewModuleHook(name, path string) *ModuleHook { + return &ModuleHook{ + CommonHook: &CommonHook{ + Name: name, + Path: path, + }, + Config: &ModuleHookConfig{}, + } +} + +func (m *ModuleHook) WithModule(module *Module) { + m.Module = module +} + +func (m *ModuleHook) WithConfig(configOutput []byte) (err error) { + err = m.Config.LoadAndValidate(configOutput) + if err != nil { + return fmt.Errorf("load module hook '%s' config: %s\nhook --config output: %s", m.Name, err.Error(), configOutput) + } + return nil +} + +func (m *ModuleHook) Order(binding BindingType) float64 { + if m.Config.HasBinding(binding) { + switch binding { + case BeforeHelm: + return m.Config.BeforeHelm.Order + case AfterHelm: + return m.Config.AfterHelm.Order + case AfterDeleteHelm: + return m.Config.AfterDeleteHelm.Order + } + } + return 0.0 +} + + +type moduleValuesMergeResult struct { + // global values with root ModuleValuesKey key + Values utils.Values + // global values under root ModuleValuesKey key + ModuleValues map[string]interface{} + ModuleValuesKey string + ValuesPatch utils.ValuesPatch + ValuesChanged bool +} + + +func (h *ModuleHook) handleModuleValuesPatch(currentValues utils.Values, valuesPatch utils.ValuesPatch) (*moduleValuesMergeResult, error) { + moduleValuesKey := utils.ModuleNameToValuesKey(h.Module.Name) + + if err := utils.ValidateHookValuesPatch(valuesPatch, moduleValuesKey); err != nil { + return nil, fmt.Errorf("merge module '%s' values failed: %s", h.Module.Name, err) + } + + newValuesRaw, valuesChanged, err := utils.ApplyValuesPatch(currentValues, valuesPatch) + if err != nil { + return nil, fmt.Errorf("merge module '%s' values failed: %s", h.Module.Name, err) + } + + result := &moduleValuesMergeResult{ + ModuleValuesKey: moduleValuesKey, + Values: utils.Values{moduleValuesKey: make(map[string]interface{})}, + ValuesChanged: valuesChanged, + ValuesPatch: valuesPatch, + } + + if moduleValuesRaw, hasKey := newValuesRaw[result.ModuleValuesKey]; hasKey { + moduleValues, ok := moduleValuesRaw.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("expected map at key '%s', got:\n%s", result.ModuleValuesKey, utils_data.YamlToString(moduleValuesRaw)) + } + result.Values[result.ModuleValuesKey] = moduleValues + result.ModuleValues = moduleValues + } + + return result, nil +} + +func (h *ModuleHook) run(bindingType BindingType, context []BindingContext) error { + moduleName := h.Module.Name + rlog.Infof("Running module hook '%s' binding '%s' ...", h.Name, bindingType) + + // Convert context for version + versionedContext := make([]interface{}, 0, len(context)) + for _, c := range context { + versionedContext = append(versionedContext, hook2.ConvertBindingContext(h.Config.Version, c.BindingContext)) + } + + moduleHookExecutor := NewHookExecutor(h, versionedContext) + patches, err := moduleHookExecutor.Run() + if err != nil { + return fmt.Errorf("module hook '%s' failed: %s", h.Name, err) + } + + configValuesPatch, has := patches[utils.ConfigMapPatch] + if has && configValuesPatch != nil{ + preparedConfigValues := utils.MergeValues( + utils.Values{utils.ModuleNameToValuesKey(moduleName): map[string]interface{}{}}, + h.moduleManager.kubeModulesConfigValues[moduleName], + ) + + configValuesPatchResult, err := h.handleModuleValuesPatch(preparedConfigValues, *configValuesPatch) + if err != nil { + return fmt.Errorf("module hook '%s': kube module config values update error: %s", h.Name, err) + } + if configValuesPatchResult.ValuesChanged { + err := h.moduleManager.kubeConfigManager.SetKubeModuleValues(moduleName, configValuesPatchResult.Values) + if err != nil { + rlog.Debugf("Module hook '%s' kube module config values stay unchanged:\n%s", utils.ValuesToString(h.moduleManager.kubeModulesConfigValues[moduleName])) + return fmt.Errorf("module hook '%s': set kube module config failed: %s", h.Name, err) + } + + h.moduleManager.kubeModulesConfigValues[moduleName] = configValuesPatchResult.Values + rlog.Debugf("Module hook '%s': kube module '%s' config values updated:\n%s", h.Name, moduleName, utils.ValuesToString(h.moduleManager.kubeModulesConfigValues[moduleName])) + } + } + + valuesPatch, has := patches[utils.MemoryValuesPatch] + if has && valuesPatch != nil { + valuesPatchResult, err := h.handleModuleValuesPatch(h.values(), *valuesPatch) + if err != nil { + return fmt.Errorf("module hook '%s': dynamic module values update error: %s", h.Name, err) + } + if valuesPatchResult.ValuesChanged { + h.moduleManager.modulesDynamicValuesPatches[moduleName] = utils.AppendValuesPatch(h.moduleManager.modulesDynamicValuesPatches[moduleName], valuesPatchResult.ValuesPatch) + rlog.Debugf("Module hook '%s': dynamic module '%s' values updated:\n%s", h.Name, moduleName, utils.ValuesToString(h.values())) + } + } + + return nil +} + +// PrepareTmpFilesForHookRun creates temporary files for hook and returns environment variables with paths +func (h *ModuleHook) PrepareTmpFilesForHookRun(context interface{}) (tmpFiles map[string]string, err error) { + tmpFiles = make(map[string]string, 0) + + tmpFiles["CONFIG_VALUES_PATH"], err = h.prepareConfigValuesJsonFile() + if err != nil { + return + } + + tmpFiles["VALUES_PATH"], err = h.prepareValuesJsonFile() + if err != nil { + return + } + + tmpFiles["BINDING_CONTEXT_PATH"], err = h.prepareBindingContextJsonFile(context) + if err != nil { + return + } + + tmpFiles["CONFIG_VALUES_JSON_PATCH_PATH"], err= h.prepareConfigValuesJsonPatchFile() + if err != nil { + return + } + + tmpFiles["VALUES_JSON_PATCH_PATH"], err = h.prepareValuesJsonPatchFile() + if err != nil { + return + } + + return +} + + +func (h *ModuleHook) configValues() utils.Values { + return h.Module.configValues() +} + +func (h *ModuleHook) values() utils.Values { + return h.Module.values() +} + +func (h *ModuleHook) prepareValuesJsonFile() (string, error) { + return h.Module.prepareValuesJsonFile() +} + +func (h *ModuleHook) prepareValuesYamlFile() (string, error) { + return h.Module.prepareValuesYamlFile() +} + +func (h *ModuleHook) prepareConfigValuesJsonFile() (string, error) { + return h.Module.prepareConfigValuesJsonFile() +} + +func (h *ModuleHook) prepareConfigValuesYamlFile() (string, error) { + return h.Module.prepareConfigValuesYamlFile() +} + +func (h *ModuleHook) prepareBindingContextJsonFile(context interface{}) (string, error) { + data, _ := json.Marshal(context) + //data := utils.MustDump(utils.DumpValuesJson(context)) + path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("%s.module-hook-%s-binding-context.json", h.Module.SafeName(), h.SafeName())) + err := dumpData(path, data) + if err != nil { + return "", err + } + + rlog.Debugf("Prepared module %s hook %s binding context:\n%s", h.Module.SafeName(), h.Name, utils_data.YamlToString(context)) + + return path, nil +} + + +func (h *ModuleHook) prepareConfigValuesJsonPatchFile() (string, error) { + path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("%s.global-hook-config-values.json-patch", h.SafeName())) + if err := CreateEmptyWritableFile(path); err != nil { + return "", err + } + return path, nil +} + +func (h *ModuleHook) prepareValuesJsonPatchFile() (string, error) { + path := filepath.Join(h.moduleManager.TempDir, fmt.Sprintf("%s.global-hook-values.json-patch", h.SafeName())) + if err := CreateEmptyWritableFile(path); err != nil { + return "", err + } + return path, nil +} diff --git a/pkg/module_manager/module_hook_config.go b/pkg/module_manager/module_hook_config.go new file mode 100644 index 00000000..1e3cf604 --- /dev/null +++ b/pkg/module_manager/module_hook_config.go @@ -0,0 +1,248 @@ +package module_manager + +import ( + "encoding/json" + "fmt" + + "github.com/go-openapi/spec" + + hook_config "github.com/flant/shell-operator/pkg/hook" + "github.com/flant/shell-operator/pkg/hook/config" +) + +// ModuleHookConfig is a structure with versioned hook configuration +type ModuleHookConfig struct { + hook_config.HookConfig + + // versioned raw config values + ModuleV0 *ModuleHookConfigV0 + ModuleV1 *ModuleHookConfigV0 + + // effective config values + BeforeHelm *BeforeHelmConfig + AfterHelm *AfterHelmConfig + AfterDeleteHelm *AfterDeleteHelmConfig +} + +type BeforeHelmConfig struct { + hook_config.CommonBindingConfig + Order float64 +} + +type AfterHelmConfig struct { + hook_config.CommonBindingConfig + Order float64 +} + +type AfterDeleteHelmConfig struct { + hook_config.CommonBindingConfig + Order float64 +} + +type ModuleHookConfigV0 struct { + BeforeHelm interface{} `json:"beforeHelm"` + AfterHelm interface{} `json:"afterHelm"` + AfterDeleteHelm interface{} `json:"afterDeleteHelm"` +} + +func GetModuleHookConfigSchema(version string) *spec.Schema { + globalHookVersion := "module-hook-"+version + if _, ok := config.Schemas[globalHookVersion]; !ok { + schema := config.Schemas[version] + switch version { + case "v1": + // add beforeHelm, afterHelm and afterDeleteHelm properties + schema += ` + beforeHelm: + type: integer + example: 10 + afterHelm: + type: integer + example: 10 + afterDeleteHelm: + type: integer + example: 10 +` + case "v0": + // add beforeHelm, afterHelm and afterDeleteHelm properties + schema += ` + beforeHelm: + type: integer + example: 10 + afterHelm: + type: integer + example: 10 + afterDeleteHelm: + type: integer + example: 10 +` + } + config.Schemas[globalHookVersion] = schema + } + + return config.GetSchema(globalHookVersion) +} + +// LoadAndValidate loads config from bytes and validate it. Returns multierror. +func (c *ModuleHookConfig) LoadAndValidate(data []byte) error { + vu := config.NewDefaultVersionedUntyped() + err := vu.Load(data) + if err != nil { + return err + } + + err = config.ValidateConfig(vu.Obj, GetModuleHookConfigSchema(vu.Version), "") + if err != nil { + return err + } + + c.Version = vu.Version + + err = c.HookConfig.ConvertAndCheck(data) + if err != nil { + return err + } + + err = c.ConvertAndCheck(data) + if err != nil { + return err + } + + return nil +} + +func (c *ModuleHookConfig) ConvertAndCheck(data []byte) error { + switch c.Version { + case "v0": + configV0 := &ModuleHookConfigV0{} + err := json.Unmarshal(data, configV0) + if err != nil { + return fmt.Errorf("unmarshal ModuleHookConfig version 0: %s", err) + } + c.ModuleV0 = configV0 + err = c.ConvertAndCheckV0() + if err != nil { + return err + } + case "v1": + configV1 := &ModuleHookConfigV0{} + err := json.Unmarshal(data, configV1) + if err != nil { + return fmt.Errorf("unmarshal ModuleHookConfig v1: %s", err) + } + c.ModuleV1 = configV1 + err = c.ConvertAndCheckV1() + if err != nil { + return err + } + default: + // NOTE: this should not happen + return fmt.Errorf("version '%s' is unsupported", c.Version) + } + + return nil +} + +func (c *ModuleHookConfig) ConvertAndCheckV0() (err error) { + c.BeforeHelm, err = c.ConvertBeforeHelm(c.ModuleV0.BeforeHelm) + if err != nil { + return err + } + c.AfterHelm, err = c.ConvertAfterHelm(c.ModuleV0.AfterHelm) + if err != nil { + return err + } + c.AfterDeleteHelm, err = c.ConvertAfterDeleteHelm(c.ModuleV0.AfterDeleteHelm) + if err != nil { + return err + } + + return nil +} + +func (c *ModuleHookConfig) ConvertAndCheckV1() (err error) { + c.BeforeHelm, err = c.ConvertBeforeHelm(c.ModuleV1.BeforeHelm) + if err != nil { + return err + } + c.AfterHelm, err = c.ConvertAfterHelm(c.ModuleV1.AfterHelm) + if err != nil { + return err + } + c.AfterDeleteHelm, err = c.ConvertAfterDeleteHelm(c.ModuleV1.AfterDeleteHelm) + if err != nil { + return err + } + + return nil +} + +func (c *ModuleHookConfig) ConvertBeforeHelm(value interface{}) (*BeforeHelmConfig, error) { + floatValue, err := hook_config.ConvertFloatForBinding(value, "beforeHelm") + if err != nil || floatValue == nil { + return nil, err + } + + res := &BeforeHelmConfig{} + res.ConfigName = ContextBindingType[BeforeHelm] + res.Order = *floatValue + return res, nil +} + +func (c *ModuleHookConfig) ConvertAfterHelm(value interface{}) (*AfterHelmConfig, error) { + floatValue, err := hook_config.ConvertFloatForBinding(value, "afterHelm") + if err != nil || floatValue == nil { + return nil, err + } + + res := &AfterHelmConfig{} + res.ConfigName = ContextBindingType[AfterHelm] + res.Order = *floatValue + return res, nil +} + +func (c *ModuleHookConfig) ConvertAfterDeleteHelm(value interface{}) (*AfterDeleteHelmConfig, error) { + floatValue, err := hook_config.ConvertFloatForBinding(value, "afterDeleteHelm") + if err != nil || floatValue == nil { + return nil, err + } + + res := &AfterDeleteHelmConfig{} + res.ConfigName = ContextBindingType[AfterDeleteHelm] + res.Order = *floatValue + return res, nil +} + + +func (c *ModuleHookConfig) Bindings() []BindingType { + res := []BindingType{} + + for _, binding := range []BindingType{OnStartup, Schedule, KubeEvents} { + if c.HookConfig.HasBinding(ShOpBindingType[binding]) { + res = append(res, binding) + } + } + + for _, binding := range []BindingType{BeforeHelm, AfterHelm, AfterDeleteHelm} { + if c.HasBinding(binding) { + res = append(res, binding) + } + } + + return res +} + +func (c *ModuleHookConfig) HasBinding(binding BindingType) bool { + if c.HookConfig.HasBinding(ShOpBindingType[binding]) { + return true + } + switch binding { + case BeforeHelm: + return c.BeforeHelm != nil + case AfterHelm: + return c.AfterHelm != nil + case AfterDeleteHelm: + return c.AfterDeleteHelm != nil + } + return false +} diff --git a/pkg/module_manager/module_hook_config_test.go b/pkg/module_manager/module_hook_config_test.go new file mode 100644 index 00000000..68bcc934 --- /dev/null +++ b/pkg/module_manager/module_hook_config_test.go @@ -0,0 +1,72 @@ +package module_manager + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_ModuleHook_Config_v0_v1(t *testing.T) { + var err error + var config *ModuleHookConfig + + tests := []struct { + name string + hookName string + data string + assertion func() + }{ + { + "load v0 module config", + "hook_v0", + `{"onStartup":10, "schedule":[{"crontab":"*/5 * * * * *"}], "afterDeleteHelm": 25}`, + func() { + if assert.NoError(t, err) { + assert.Len(t, config.Schedules, 1) + assert.True(t, config.HasBinding(OnStartup)) + assert.True(t, config.HasBinding(Schedule)) + assert.True(t, config.HasBinding(AfterDeleteHelm)) + assert.False(t, config.HasBinding(KubeEvents)) + assert.False(t, config.HasBinding(AfterHelm)) + assert.False(t, config.HasBinding(BeforeHelm)) + } + }, + }, + { + "load v1 module config", + "hook_v1", + `{"configVersion": "v1", "onStartup":10, "kubernetes":[{"kind":"Pod", "watchEvent":["Added"]}], "beforeHelm":98}`, + func() { + if assert.NoError(t, err) { + assert.Len(t, config.OnKubernetesEvents, 1) + assert.Len(t, config.Schedules, 0) + assert.True(t, config.HasBinding(OnStartup)) + assert.True(t, config.HasBinding(KubeEvents)) + assert.True(t, config.HasBinding(BeforeHelm)) + assert.False(t, config.HasBinding(Schedule)) + assert.False(t, config.HasBinding(AfterDeleteHelm)) + assert.False(t, config.HasBinding(AfterHelm)) + } + + }, + }, + { + "load v1 bad module config", + "hook_v1", + `{"configVersion": "v1", "onStartuppp":10, "kubernetes":[{"kind":"Pod", "watchEvent":["Added"]}], "beforeHelm":98}`, + func() { + assert.Error(t, err) + t.Logf("expected error: %v", err) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config = &ModuleHookConfig{} + err = config.LoadAndValidate([]byte(test.data)) + test.assertion() + }) + } +} + diff --git a/pkg/module_manager/module_manager.go b/pkg/module_manager/module_manager.go index 7bec90cc..46d2d7d1 100644 --- a/pkg/module_manager/module_manager.go +++ b/pkg/module_manager/module_manager.go @@ -1,7 +1,6 @@ package module_manager import ( - "encoding/json" "fmt" "reflect" "sort" @@ -9,7 +8,7 @@ import ( "github.com/romana/rlog" - utils_checksum "github.com/flant/shell-operator/pkg/utils/checksum" + hook_config "github.com/flant/shell-operator/pkg/hook" "github.com/flant/addon-operator/pkg/helm" "github.com/flant/addon-operator/pkg/kube_config_manager" @@ -32,7 +31,7 @@ type ModuleManager interface { RunModule(moduleName string, onStartup bool) error RunGlobalHook(hookName string, binding BindingType, bindingContext []BindingContext) error RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext) error - InitModuleHooks(module *Module) error + RegisterModuleHooks(module *Module) error Retry() WithDirectories(modulesDir string, globalHooksDir string, tempDir string) ModuleManager WithKubeConfigManager(kubeConfigManager kube_config_manager.KubeConfigManager) ModuleManager @@ -145,13 +144,15 @@ var ContextBindingType = map[BindingType]string{ KubeEvents: "onKubernetesEvent", } +var ShOpBindingType = map[BindingType]hook_config.BindingType{ + OnStartup: hook_config.OnStartup, + Schedule: hook_config.Schedule, + KubeEvents: hook_config.OnKubernetesEvent, +} + // BindingContext is a json with additional info for schedule and onKubeEvent hooks type BindingContext struct { - Binding string `json:"binding"` - ResourceEvent string `json:"resourceEvent,omitempty"` - ResourceNamespace string `json:"resourceNamespace,omitempty"` - ResourceKind string `json:"resourceKind,omitempty"` - ResourceName string `json:"resourceName,omitempty"` + hook_config.BindingContext } // EventType are events for the main loop. @@ -385,9 +386,9 @@ func (mm *MainModuleManager) calculateEnabledModulesByConfig(moduleConfigs kube_ for moduleName, module := range mm.allModulesByName { kubeConfig, hasKubeConfig := moduleConfigs[moduleName] if hasKubeConfig { - isEnabled := mergeEnabled(module.CommonStaticConfig.IsEnabled, - module.StaticConfig.IsEnabled, - kubeConfig.IsEnabled) + isEnabled := mergeEnabled(module.CommonStaticConfig.IsEnabled, + module.StaticConfig.IsEnabled, + kubeConfig.IsEnabled) if isEnabled { enabled = append(enabled, moduleName) @@ -422,7 +423,7 @@ func (mm *MainModuleManager) calculateEnabledModulesByConfig(moduleConfigs kube_ func (mm *MainModuleManager) Init() error { rlog.Debug("INIT: MODULE_MANAGER") - if err := mm.initGlobalHooks(); err != nil { + if err := mm.RegisterGlobalHooks(); err != nil { return err } @@ -565,7 +566,7 @@ func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err er } for _, moduleName := range enabledModules { - if err = mm.InitModuleHooks(mm.allModulesByName[moduleName]); err != nil { + if err = mm.RegisterModuleHooks(mm.allModulesByName[moduleName]); err != nil { return nil, err } } @@ -582,7 +583,6 @@ func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err er state.ModulesToDisable = utils.ListIntersection(state.ModulesToDisable, releasedModules) state.ModulesToDisable = utils.SortReverseByReference(state.ModulesToDisable, mm.allModulesNamesInOrder) - rlog.Debugf("DISCOVER state results:\n"+ " mm.enabledModulesByConfig: %v\n"+ " EnabledModules: %v\n"+ @@ -640,7 +640,7 @@ func (mm *MainModuleManager) GetGlobalHooksInOrder(bindingType BindingType) []st } sort.Slice(globalHooks[:], func(i, j int) bool { - return globalHooks[i].OrderByBinding[bindingType] < globalHooks[j].OrderByBinding[bindingType] + return globalHooks[i].Order(bindingType) < globalHooks[j].Order(bindingType) }) var globalHooksNames []string @@ -667,7 +667,7 @@ func (mm *MainModuleManager) GetModuleHooksInOrder(moduleName string, bindingTyp } sort.Slice(moduleBindingHooks[:], func(i, j int) bool { - return moduleBindingHooks[i].OrderByBinding[bindingType] < moduleBindingHooks[j].OrderByBinding[bindingType] + return moduleBindingHooks[i].Order(bindingType) < moduleBindingHooks[j].Order(bindingType) }) var moduleHooksNames []string @@ -689,8 +689,8 @@ func (mm *MainModuleManager) DeleteModule(moduleName string) error { return err } - // remove hooks structures - mm.removeModuleHooks(moduleName) + // remove module hooks from indexes + delete(mm.modulesHooksOrderByName, moduleName) return nil } @@ -709,21 +709,13 @@ func (mm *MainModuleManager) RunModule(moduleName string, onStartup bool) error return nil } -func valuesChecksum(valuesArr ...utils.Values) (string, error) { - valuesJson, err := json.Marshal(utils.MergeValues(valuesArr...)) - if err != nil { - return "", err - } - return utils_checksum.CalculateChecksum(string(valuesJson)), nil -} - func (mm *MainModuleManager) RunGlobalHook(hookName string, binding BindingType, bindingContext []BindingContext) error { globalHook, err := mm.GetGlobalHook(hookName) if err != nil { return err } - oldValuesChecksum, err := valuesChecksum(globalHook.values()) + oldValuesChecksum, err := utils.ValuesChecksum(globalHook.values()) if err != nil { return err } @@ -732,7 +724,7 @@ func (mm *MainModuleManager) RunGlobalHook(hookName string, binding BindingType, return err } - newValuesChecksum, err := valuesChecksum(globalHook.values()) + newValuesChecksum, err := utils.ValuesChecksum(globalHook.values()) if err != nil { return err } @@ -753,7 +745,7 @@ func (mm *MainModuleManager) RunModuleHook(hookName string, binding BindingType, return err } - oldValuesChecksum, err := valuesChecksum(moduleHook.values()) + oldValuesChecksum, err := utils.ValuesChecksum(moduleHook.values()) if err != nil { return err } @@ -762,7 +754,7 @@ func (mm *MainModuleManager) RunModuleHook(hookName string, binding BindingType, return err } - newValuesChecksum, err := valuesChecksum(moduleHook.values()) + newValuesChecksum, err := utils.ValuesChecksum(moduleHook.values()) if err != nil { return err } diff --git a/pkg/module_manager/module_manager_mock.go b/pkg/module_manager/module_manager_mock.go new file mode 100644 index 00000000..70764523 --- /dev/null +++ b/pkg/module_manager/module_manager_mock.go @@ -0,0 +1,153 @@ +package module_manager + +import "github.com/flant/addon-operator/pkg/kube_config_manager" + +type ModuleManagerMockFns struct { + Init func() error + Run func() + DiscoverModulesState func() (*ModulesState, error) + GetModule func(name string) (*Module, error) + GetModuleNamesInOrder func() []string + GetGlobalHook func(name string) (*GlobalHook, error) + GetModuleHook func(name string) (*ModuleHook, error) + GetGlobalHooksInOrder func(bindingType BindingType) []string + GetModuleHooksInOrder func(moduleName string, bindingType BindingType) ([]string, error) + DeleteModule func(moduleName string) error + RunModule func(moduleName string, onStartup bool) error + RunGlobalHook func(hookName string, binding BindingType, bindingContext []BindingContext) error + RunModuleHook func(hookName string, binding BindingType, bindingContext []BindingContext) error + Retry func() + WithDirectories func(modulesDir string, globalHooksDir string, tempDir string) ModuleManager + WithKubeConfigManager func(kubeConfigManager kube_config_manager.KubeConfigManager) ModuleManager + RegisterModuleHooks func(*Module) error +} + + +func NewModuleManagerMock(fns ModuleManagerMockFns) ModuleManager { + return &ModuleManagerMock{ + Fns: fns, + } +} + +type ModuleManagerMock struct { + Fns ModuleManagerMockFns +} + +func (m *ModuleManagerMock) Init() error { + if m.Fns.Init != nil { + return m.Fns.Init() + } + panic("implement me") +} + +func (m *ModuleManagerMock) Run() { + if m.Fns.Run != nil { + m.Fns.Run() + } + panic("implement me") +} + +func (m *ModuleManagerMock) DiscoverModulesState() (*ModulesState, error) { + if m.Fns.DiscoverModulesState != nil { + return m.Fns.DiscoverModulesState() + } + panic("implement me") +} + +func (m *ModuleManagerMock) GetModule(name string) (*Module, error) { + if m.Fns.GetModule != nil { + return m.Fns.GetModule(name) + } + panic("implement me") +} + +func (m *ModuleManagerMock) GetModuleNamesInOrder() []string { + if m.Fns.GetModuleNamesInOrder != nil { + return m.Fns.GetModuleNamesInOrder() + } + panic("implement me") +} + +func (m *ModuleManagerMock) GetGlobalHook(name string) (*GlobalHook, error) { + if m.Fns.GetGlobalHook != nil { + return m.Fns.GetGlobalHook(name) + } + panic("implement me") +} + +func (m *ModuleManagerMock) GetModuleHook(name string) (*ModuleHook, error) { + if m.Fns.GetModuleHook != nil { + return m.Fns.GetModuleHook(name) + } + panic("implement me") +} + +func (m *ModuleManagerMock) GetGlobalHooksInOrder(bindingType BindingType) []string { + if m.Fns.GetGlobalHooksInOrder != nil { + return m.Fns.GetGlobalHooksInOrder(bindingType) + } + panic("implement me") +} + +func (m *ModuleManagerMock) GetModuleHooksInOrder(moduleName string, bindingType BindingType) ([]string, error) { + if m.Fns.GetModuleHooksInOrder != nil { + return m.Fns.GetModuleHooksInOrder(moduleName, bindingType) + } + panic("implement me") +} + +func (m *ModuleManagerMock) DeleteModule(moduleName string) error { + if m.Fns.DeleteModule != nil { + return m.Fns.DeleteModule(moduleName) + } + panic("implement me") +} + +func (m *ModuleManagerMock) RunModule(moduleName string, onStartup bool) error { + if m.Fns.RunModule != nil { + return m.Fns.RunModule(moduleName, onStartup) + } + panic("implement me") +} + +func (m *ModuleManagerMock) RunGlobalHook(hookName string, binding BindingType, bindingContext []BindingContext) error { + if m.Fns.RunGlobalHook != nil { + return m.Fns.RunGlobalHook(hookName, binding, bindingContext) + } + panic("implement me") +} + +func (m *ModuleManagerMock) RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext) error { + if m.Fns.RunModuleHook != nil { + return m.Fns.RunModuleHook(hookName, binding, bindingContext) + } + panic("implement me") +} + +func (m *ModuleManagerMock) Retry() { + if m.Fns.Retry != nil { + m.Fns.Retry() + } + panic("implement me") +} + +func (m *ModuleManagerMock) WithDirectories(modulesDir string, globalHooksDir string, tempDir string) ModuleManager { + if m.Fns.WithDirectories != nil { + return m.Fns.WithDirectories(modulesDir, globalHooksDir, tempDir) + } + panic("implement me") +} + +func (m *ModuleManagerMock) WithKubeConfigManager(kubeConfigManager kube_config_manager.KubeConfigManager) ModuleManager { + if m.Fns.WithKubeConfigManager != nil { + return m.Fns.WithKubeConfigManager(kubeConfigManager) + } + panic("implement me") +} + +func (m *ModuleManagerMock) RegisterModuleHooks(module *Module) error { + if m.Fns.RegisterModuleHooks != nil { + return m.Fns.RegisterModuleHooks(module) + } + panic("implement me") +} diff --git a/pkg/module_manager/module_manager_test.go b/pkg/module_manager/module_manager_test.go index 36019cf8..a926ca2f 100644 --- a/pkg/module_manager/module_manager_test.go +++ b/pkg/module_manager/module_manager_test.go @@ -7,18 +7,14 @@ import ( "reflect" "testing" - "github.com/ghodss/yaml" "github.com/romana/rlog" "github.com/stretchr/testify/assert" - - "k8s.io/client-go/kubernetes/fake" - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" "github.com/flant/shell-operator/pkg/kube" - "github.com/flant/shell-operator/pkg/kube_events_manager" - "github.com/flant/shell-operator/pkg/schedule_manager" utils_file "github.com/flant/shell-operator/pkg/utils/file" + "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/fake" "github.com/flant/addon-operator/pkg/app" "github.com/flant/addon-operator/pkg/helm" @@ -46,7 +42,7 @@ func initModuleManager(t *testing.T, mm *MainModuleManager, configPath string) { t.Fatal(err) } - if err := mm.initGlobalHooks(); err != nil { + if err := mm.RegisterGlobalHooks(); err != nil { t.Fatal(err) } @@ -299,185 +295,185 @@ func Test_MainModuleManager_Get_Module(t *testing.T) { } } -func Test_MainModuleManager_Get_ModuleHook(t *testing.T) { - t.SkipNow() - mm := NewMainModuleManager() - - initModuleManager(t, mm, "get__module_hook") - - var moduleHook *ModuleHook - var err error - - tests := []struct{ - name string - hookName string - testFn func() - } { - { - "module-hook-all-bindings", - "000-all-bindings/hooks/all", - func() { - expectedHook := &ModuleHook{ - &CommonHook{ - "000-all-bindings/hooks/all", - filepath.Join(mm.ModulesDir, "000-all-bindings/hooks/all"), - []BindingType{BeforeHelm, AfterHelm, AfterDeleteHelm, OnStartup, Schedule, KubeEvents}, - map[BindingType]float64 { - BeforeHelm: 1.0, - AfterHelm: 1.0, - AfterDeleteHelm: 1.0, - OnStartup: 1.0, - }, - mm, - }, - &Module{}, - &ModuleHookConfig{ - HookConfig{ - 1.0, - []schedule_manager.ScheduleConfig{ - { - Crontab: "* * * * *", - AllowFailure: true, - }, - }, - []kube_events_manager.OnKubernetesEventConfig{ - { - EventTypes: []kube_events_manager.OnKubernetesEventType{kube_events_manager.KubernetesEventOnAdd}, - Kind: "configmap", - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "component": "component1", - }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "tier", - Operator: "In", - Values: []string{"cache"}, - }, - }, - }, - NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ - MatchNames: []string{"namespace1"}, - Any: false, - }, - JqFilter: ".items[] | del(.metadata, .field1)", - AllowFailure: true, - }, - { - EventTypes: []kube_events_manager.OnKubernetesEventType{ - kube_events_manager.KubernetesEventOnAdd, - kube_events_manager.KubernetesEventOnUpdate, - kube_events_manager.KubernetesEventOnDelete, - }, - Kind: "namespace", - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "component": "component2", - }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "tier", - Operator: "In", - Values: []string{"cache"}, - }, - }, - }, - NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ - MatchNames: []string{"namespace2"}, - Any: false, - }, - JqFilter: ".items[] | del(.metadata, .field2)", - AllowFailure: true, - }, - { - EventTypes: []kube_events_manager.OnKubernetesEventType{ - kube_events_manager.KubernetesEventOnAdd, - kube_events_manager.KubernetesEventOnUpdate, - kube_events_manager.KubernetesEventOnDelete, - }, - Kind: "pod", - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "component": "component3", - }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "tier", - Operator: "In", - Values: []string{"cache"}, - }, - }, - }, - NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ - MatchNames: nil, - Any: true, - }, - JqFilter: ".items[] | del(.metadata, .field3)", - AllowFailure: true, - }, - }, - }, - 1.0, - 1.0, - 1.0, - }, - } - if assert.NoError(t, err) { - moduleHook.Module = &Module{} - assert.Equal(t, expectedHook, moduleHook) - } - }, - }, - { - "nested-module-hook", - "100-nested-hooks/hooks/sub/sub/nested-before-helm", - func() { - expectedHook := &ModuleHook{ - &CommonHook{ - "100-nested-hooks/hooks/sub/sub/nested-before-helm", - filepath.Join(mm.ModulesDir, "100-nested-hooks/hooks/sub/sub/nested-before-helm"), - []BindingType{BeforeHelm}, - map[BindingType]float64 { - BeforeHelm: 1.0, - }, - mm, - }, - &Module{}, - &ModuleHookConfig{ - HookConfig{ - OnStartup: nil, - Schedule: nil, - OnKubernetesEvent: nil, - }, - 1.0, - nil, - nil, - }, - } - if assert.NoError(t, err) { - moduleHook.Module = &Module{} - assert.Equal(t, expectedHook, moduleHook) - } - }, - }, - { - "error-on-non-existent-module-hook", - "non-existent", - func() { - assert.Error(t, err) - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - moduleHook = nil - err = nil - moduleHook, err = mm.GetModuleHook(test.hookName) - test.testFn() - }) - } -} +//func Test_MainModuleManager_Get_ModuleHook(t *testing.T) { +// t.SkipNow() +// mm := NewMainModuleManager() +// +// initModuleManager(t, mm, "get__module_hook") +// +// var moduleHook *ModuleHook +// var err error +// +// tests := []struct{ +// name string +// hookName string +// testFn func() +// } { +// { +// "module-hook-all-bindings", +// "000-all-bindings/hooks/all", +// func() { +// expectedHook := &ModuleHook{ +// &CommonHook{ +// "000-all-bindings/hooks/all", +// filepath.Join(mm.ModulesDir, "000-all-bindings/hooks/all"), +// []BindingType{BeforeHelm, AfterHelm, AfterDeleteHelm, OnStartup, Schedule, KubeEvents}, +// map[BindingType]float64 { +// BeforeHelm: 1.0, +// AfterHelm: 1.0, +// AfterDeleteHelm: 1.0, +// OnStartup: 1.0, +// }, +// mm, +// }, +// &Module{}, +// &ModuleHookConfig{ +// HookConfig{ +// 1.0, +// []schedule_manager.ScheduleConfig{ +// { +// Crontab: "* * * * *", +// AllowFailure: true, +// }, +// }, +// []kube_events_manager.OnKubernetesEventConfig{ +// { +// EventTypes: []kube_events_manager.OnKubernetesEventType{kube_events_manager.KubernetesEventOnAdd}, +// Kind: "configmap", +// Selector: &metav1.LabelSelector{ +// MatchLabels: map[string]string{ +// "component": "component1", +// }, +// MatchExpressions: []metav1.LabelSelectorRequirement{ +// { +// Key: "tier", +// Operator: "In", +// Values: []string{"cache"}, +// }, +// }, +// }, +// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ +// MatchNames: []string{"namespace1"}, +// Any: false, +// }, +// JqFilter: ".items[] | del(.metadata, .field1)", +// AllowFailure: true, +// }, +// { +// EventTypes: []kube_events_manager.OnKubernetesEventType{ +// kube_events_manager.KubernetesEventOnAdd, +// kube_events_manager.KubernetesEventOnUpdate, +// kube_events_manager.KubernetesEventOnDelete, +// }, +// Kind: "namespace", +// Selector: &metav1.LabelSelector{ +// MatchLabels: map[string]string{ +// "component": "component2", +// }, +// MatchExpressions: []metav1.LabelSelectorRequirement{ +// { +// Key: "tier", +// Operator: "In", +// Values: []string{"cache"}, +// }, +// }, +// }, +// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ +// MatchNames: []string{"namespace2"}, +// Any: false, +// }, +// JqFilter: ".items[] | del(.metadata, .field2)", +// AllowFailure: true, +// }, +// { +// EventTypes: []kube_events_manager.OnKubernetesEventType{ +// kube_events_manager.KubernetesEventOnAdd, +// kube_events_manager.KubernetesEventOnUpdate, +// kube_events_manager.KubernetesEventOnDelete, +// }, +// Kind: "pod", +// Selector: &metav1.LabelSelector{ +// MatchLabels: map[string]string{ +// "component": "component3", +// }, +// MatchExpressions: []metav1.LabelSelectorRequirement{ +// { +// Key: "tier", +// Operator: "In", +// Values: []string{"cache"}, +// }, +// }, +// }, +// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ +// MatchNames: nil, +// Any: true, +// }, +// JqFilter: ".items[] | del(.metadata, .field3)", +// AllowFailure: true, +// }, +// }, +// }, +// 1.0, +// 1.0, +// 1.0, +// }, +// } +// if assert.NoError(t, err) { +// moduleHook.Module = &Module{} +// assert.Equal(t, expectedHook, moduleHook) +// } +// }, +// }, +// { +// "nested-module-hook", +// "100-nested-hooks/hooks/sub/sub/nested-before-helm", +// func() { +// expectedHook := &ModuleHook{ +// &CommonHook{ +// "100-nested-hooks/hooks/sub/sub/nested-before-helm", +// filepath.Join(mm.ModulesDir, "100-nested-hooks/hooks/sub/sub/nested-before-helm"), +// []BindingType{BeforeHelm}, +// map[BindingType]float64 { +// BeforeHelm: 1.0, +// }, +// mm, +// }, +// &Module{}, +// &ModuleHookConfig{ +// HookConfig{ +// OnStartup: nil, +// Schedule: nil, +// OnKubernetesEvent: nil, +// }, +// 1.0, +// nil, +// nil, +// }, +// } +// if assert.NoError(t, err) { +// moduleHook.Module = &Module{} +// assert.Equal(t, expectedHook, moduleHook) +// } +// }, +// }, +// { +// "error-on-non-existent-module-hook", +// "non-existent", +// func() { +// assert.Error(t, err) +// }, +// }, +// } +// +// for _, test := range tests { +// t.Run(test.name, func(t *testing.T) { +// moduleHook = nil +// err = nil +// moduleHook, err = mm.GetModuleHook(test.hookName) +// test.testFn() +// }) +// } +//} func Test_MainModuleManager_Get_ModuleHooksInOrder(t *testing.T) { helm.Client = &helm.MockHelmClient{} @@ -791,177 +787,177 @@ func Test_MainModuleManager_RunModuleHook(t *testing.T) { } } -func Test_MainModuleManager_Get_GlobalHook(t *testing.T) { - mm := NewMainModuleManager() - - initModuleManager(t, mm, "get__global_hook") - - var globalHook *GlobalHook - var err error - - tests := []struct { - name string - hookName string - testFn func() - }{ - { - "global-hook-with-all-bindings", - "000-all-bindings/all", - func() { - expectedHook := &GlobalHook{ - &CommonHook{ - "000-all-bindings/all", - filepath.Join(mm.GlobalHooksDir, "000-all-bindings/all"), - []BindingType{BeforeAll, AfterAll, OnStartup, Schedule, KubeEvents}, - map[BindingType]float64{ - BeforeAll: 1.0, - AfterAll: 1.0, - OnStartup: 1.0, - }, - mm, - }, - &GlobalHookConfig{ - HookConfig{ - 1.0, - []schedule_manager.ScheduleConfig{ - { - Crontab: "* * * * *", - AllowFailure: true, - }, - }, - []kube_events_manager.OnKubernetesEventConfig{ - { - EventTypes: []kube_events_manager.OnKubernetesEventType{kube_events_manager.KubernetesEventOnAdd}, - Kind: "configmap", - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "component": "component1", - }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "tier", - Operator: "In", - Values: []string{"cache"}, - }, - }, - }, - NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ - MatchNames: []string{"namespace1"}, - Any: false, - }, - JqFilter: ".items[] | del(.metadata, .field1)", - AllowFailure: true, - }, - { - EventTypes: []kube_events_manager.OnKubernetesEventType{ - kube_events_manager.KubernetesEventOnAdd, - kube_events_manager.KubernetesEventOnUpdate, - kube_events_manager.KubernetesEventOnDelete, - }, - Kind: "namespace", - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "component": "component2", - }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "tier", - Operator: "In", - Values: []string{"cache"}, - }, - }, - }, - NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ - MatchNames: []string{"namespace2"}, - Any: false, - }, - JqFilter: ".items[] | del(.metadata, .field2)", - AllowFailure: true, - }, - { - EventTypes: []kube_events_manager.OnKubernetesEventType{ - kube_events_manager.KubernetesEventOnAdd, - kube_events_manager.KubernetesEventOnUpdate, - kube_events_manager.KubernetesEventOnDelete, - }, - Kind: "pod", - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "component": "component3", - }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "tier", - Operator: "In", - Values: []string{"cache"}, - }, - }, - }, - NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ - MatchNames: nil, - Any: true, - }, - JqFilter: ".items[] | del(.metadata, .field3)", - AllowFailure: true, - }, - }, - }, - 1.0, - 1.0, - }, - } - - if assert.NoError(t, err) { - assert.Equal(t, expectedHook, globalHook) - } - }, - }, - { - "global-hook-nested", - "100-nested-hook/sub/sub/nested-before-all", - func() { - expectedHook := &GlobalHook{ - &CommonHook { - "100-nested-hook/sub/sub/nested-before-all", - filepath.Join(mm.GlobalHooksDir, "100-nested-hook/sub/sub/nested-before-all"), - []BindingType{BeforeAll}, - map[BindingType]float64{ - BeforeAll: 1.0, - }, - mm, - }, - &GlobalHookConfig{ - HookConfig{ - nil, - nil, - nil, - }, - 1.0, - nil, - }, - } - if assert.NoError(t, err) { - assert.Equal(t, expectedHook, globalHook) - } - }, - }, - { - "error-if-hook-not-registered", - "non-existent", - func(){ - assert.Error(t, err) - assert.Nil(t, globalHook) - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - globalHook, err = mm.GetGlobalHook(test.hookName) - test.testFn() - }) - } -} +//func Test_MainModuleManager_Get_GlobalHook(t *testing.T) { +// mm := NewMainModuleManager() +// +// initModuleManager(t, mm, "get__global_hook") +// +// var globalHook *GlobalHook +// var err error +// +// tests := []struct { +// name string +// hookName string +// testFn func() +// }{ +// { +// "global-hook-with-all-bindings", +// "000-all-bindings/all", +// func() { +// expectedHook := &GlobalHook{ +// &CommonHook{ +// "000-all-bindings/all", +// filepath.Join(mm.GlobalHooksDir, "000-all-bindings/all"), +// []BindingType{BeforeAll, AfterAll, OnStartup, Schedule, KubeEvents}, +// map[BindingType]float64{ +// BeforeAll: 1.0, +// AfterAll: 1.0, +// OnStartup: 1.0, +// }, +// mm, +// }, +// &GlobalHookConfig{ +// HookConfig{ +// 1.0, +// []schedule_manager.ScheduleConfig{ +// { +// Crontab: "* * * * *", +// AllowFailure: true, +// }, +// }, +// []kube_events_manager.OnKubernetesEventConfig{ +// { +// EventTypes: []kube_events_manager.OnKubernetesEventType{kube_events_manager.KubernetesEventOnAdd}, +// Kind: "configmap", +// Selector: &metav1.LabelSelector{ +// MatchLabels: map[string]string{ +// "component": "component1", +// }, +// MatchExpressions: []metav1.LabelSelectorRequirement{ +// { +// Key: "tier", +// Operator: "In", +// Values: []string{"cache"}, +// }, +// }, +// }, +// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ +// MatchNames: []string{"namespace1"}, +// Any: false, +// }, +// JqFilter: ".items[] | del(.metadata, .field1)", +// AllowFailure: true, +// }, +// { +// EventTypes: []kube_events_manager.OnKubernetesEventType{ +// kube_events_manager.KubernetesEventOnAdd, +// kube_events_manager.KubernetesEventOnUpdate, +// kube_events_manager.KubernetesEventOnDelete, +// }, +// Kind: "namespace", +// Selector: &metav1.LabelSelector{ +// MatchLabels: map[string]string{ +// "component": "component2", +// }, +// MatchExpressions: []metav1.LabelSelectorRequirement{ +// { +// Key: "tier", +// Operator: "In", +// Values: []string{"cache"}, +// }, +// }, +// }, +// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ +// MatchNames: []string{"namespace2"}, +// Any: false, +// }, +// JqFilter: ".items[] | del(.metadata, .field2)", +// AllowFailure: true, +// }, +// { +// EventTypes: []kube_events_manager.OnKubernetesEventType{ +// kube_events_manager.KubernetesEventOnAdd, +// kube_events_manager.KubernetesEventOnUpdate, +// kube_events_manager.KubernetesEventOnDelete, +// }, +// Kind: "pod", +// Selector: &metav1.LabelSelector{ +// MatchLabels: map[string]string{ +// "component": "component3", +// }, +// MatchExpressions: []metav1.LabelSelectorRequirement{ +// { +// Key: "tier", +// Operator: "In", +// Values: []string{"cache"}, +// }, +// }, +// }, +// NamespaceSelector: &kube_events_manager.KubeNamespaceSelector{ +// MatchNames: nil, +// Any: true, +// }, +// JqFilter: ".items[] | del(.metadata, .field3)", +// AllowFailure: true, +// }, +// }, +// }, +// 1.0, +// 1.0, +// }, +// } +// +// if assert.NoError(t, err) { +// assert.Equal(t, expectedHook, globalHook) +// } +// }, +// }, +// { +// "global-hook-nested", +// "100-nested-hook/sub/sub/nested-before-all", +// func() { +// expectedHook := &GlobalHook{ +// &CommonHook { +// "100-nested-hook/sub/sub/nested-before-all", +// filepath.Join(mm.GlobalHooksDir, "100-nested-hook/sub/sub/nested-before-all"), +// []BindingType{BeforeAll}, +// map[BindingType]float64{ +// BeforeAll: 1.0, +// }, +// mm, +// }, +// &GlobalHookConfig{ +// HookConfig{ +// nil, +// nil, +// nil, +// }, +// 1.0, +// nil, +// }, +// } +// if assert.NoError(t, err) { +// assert.Equal(t, expectedHook, globalHook) +// } +// }, +// }, +// { +// "error-if-hook-not-registered", +// "non-existent", +// func(){ +// assert.Error(t, err) +// assert.Nil(t, globalHook) +// }, +// }, +// } +// +// for _, test := range tests { +// t.Run(test.name, func(t *testing.T) { +// globalHook, err = mm.GetGlobalHook(test.hookName) +// test.testFn() +// }) +// } +//} func Test_MainModuleManager_Get_GlobalHooksInOrder(t *testing.T) { helm.Client = &helm.MockHelmClient{} diff --git a/pkg/module_manager/utils.go b/pkg/module_manager/utils.go new file mode 100644 index 00000000..24a16459 --- /dev/null +++ b/pkg/module_manager/utils.go @@ -0,0 +1,14 @@ +package module_manager + +import "os" + +func CreateEmptyWritableFile(filePath string) error { + file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + return nil + } + + _ = file.Close() + return nil +} + diff --git a/pkg/utils/module_config.go b/pkg/utils/module_config.go index c566b905..3ab87305 100644 --- a/pkg/utils/module_config.go +++ b/pkg/utils/module_config.go @@ -3,8 +3,9 @@ package utils import ( "fmt" + "gopkg.in/yaml.v2" + utils_checksum "github.com/flant/shell-operator/pkg/utils/checksum" - "github.com/go-yaml/yaml" ) var ModuleEnabled = true diff --git a/pkg/utils/values.go b/pkg/utils/values.go index 7e31fd0b..8e09ba2f 100644 --- a/pkg/utils/values.go +++ b/pkg/utils/values.go @@ -11,12 +11,12 @@ import ( "github.com/romana/rlog" "github.com/evanphx/json-patch" - ghodssyaml "github.com/ghodss/yaml" - "github.com/go-yaml/yaml" "github.com/peterbourgon/mergemap" "github.com/segmentio/go-camelcase" + "gopkg.in/yaml.v2" + k8syaml "sigs.k8s.io/yaml" - utils_data "github.com/flant/shell-operator/pkg/utils/data" + utils_checksum "github.com/flant/shell-operator/pkg/utils/checksum" ) const ( @@ -113,7 +113,7 @@ func NewValuesFromBytes(data []byte) (Values, error) { func NewValues(data map[interface{}]interface{}) (Values, error) { values, err := FormatValues(data) if err != nil { - return nil, fmt.Errorf("cannot cast data to JSON compatible format: %s:\n%s", err, utils_data.YamlToString(data)) + return nil, fmt.Errorf("cannot cast data to JSON compatible format: %s:\n%s", err, ValuesToString(values)) } return values, nil @@ -125,7 +125,7 @@ func FormatValues(someValues map[interface{}]interface{}) (Values, error) { return nil, err } - jsonDoc, err := ghodssyaml.YAMLToJSON(yamlDoc) + jsonDoc, err := k8syaml.YAMLToJSON(yamlDoc) if err != nil { return nil, err } @@ -253,6 +253,24 @@ func ApplyJsonPatchToValues(values Values, patch jsonpatch.Patch) (Values, error return resValues, nil } +func ValidateHookValuesPatch(valuesPatch ValuesPatch, acceptableKey string) error { + for _, op := range valuesPatch.Operations { + if op.Op == "replace" { + return fmt.Errorf("unsupported patch operation '%s': '%s'", op.Op, op.ToString()) + } + + pathParts := strings.Split(op.Path, "/") + if len(pathParts) > 1 { + affectedKey := pathParts[1] + if affectedKey != acceptableKey { + return fmt.Errorf("unacceptable patch operation path '%s' (only '%s' accepted): '%s'", affectedKey, acceptableKey, op.ToString()) + } + } + } + + return nil +} + func MergeValues(values ...Values) Values { res := make(Values) @@ -264,9 +282,23 @@ func MergeValues(values ...Values) Values { } func ValuesToString(values Values) string { - return utils_data.YamlToString(values) + valuesYaml, err := yaml.Marshal(&values) + if err != nil { + panic(fmt.Sprintf("Cannot dump data to YAML: \n%#v\n error: %s", values, err)) + } + return string(valuesYaml) +} + + +func ValuesChecksum(valuesArr ...Values) (string, error) { + valuesJson, err := json.Marshal(MergeValues(valuesArr...)) + if err != nil { + return "", err + } + return utils_checksum.CalculateChecksum(string(valuesJson)), nil } + func MustDump(data []byte, err error) []byte { if err != nil { panic(err) From 9dfa450934b4d828ea2232d4a97caa75c4dd0904 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Thu, 17 Oct 2019 09:06:29 +0300 Subject: [PATCH 2/8] feat: json logging --- cmd/addon-operator/main.go | 5 +- go.mod | 5 +- go.sum | 10 +- pkg/addon-operator/operator.go | 137 ++++++++++-------- pkg/addon-operator/start.go | 9 +- pkg/helm/helm.go | 94 ++++++------ pkg/helm/helm_test.go | 23 ++- pkg/helm/tiller.go | 14 +- .../kube_config_manager.go | 38 ++--- pkg/kube_config_manager/module_kube_config.go | 5 +- pkg/module_manager/global_hook.go | 20 +-- pkg/module_manager/hook.go | 20 +-- .../hook/kube_event/hooks_controller.go | 28 +--- .../hook/schedule/hooks_controller.go | 16 +- pkg/module_manager/hook_executor.go | 12 +- pkg/module_manager/module.go | 76 +++++----- pkg/module_manager/module_hook.go | 12 +- pkg/module_manager/module_manager.go | 80 +++++----- pkg/module_manager/module_manager_test.go | 37 +++-- pkg/task/tasks_queue_dumper.go | 8 +- pkg/utils/values.go | 4 +- 21 files changed, 357 insertions(+), 296 deletions(-) diff --git a/cmd/addon-operator/main.go b/cmd/addon-operator/main.go index ddad1347..bd5fb49a 100644 --- a/cmd/addon-operator/main.go +++ b/cmd/addon-operator/main.go @@ -4,9 +4,10 @@ import ( "fmt" "os" - "github.com/flant/shell-operator/pkg/executor" "gopkg.in/alecthomas/kingpin.v2" + shell_operator_app "github.com/flant/shell-operator/pkg/app" + "github.com/flant/shell-operator/pkg/executor" utils_signal "github.com/flant/shell-operator/pkg/utils/signal" operator "github.com/flant/addon-operator/pkg/addon-operator" @@ -18,6 +19,7 @@ func main() { // global defaults app.SetupGlobalSettings(kpApp) + shell_operator_app.SetupGlobalSettings(kpApp) // print version kpApp.Command("version", "Show version.").Action(func(c *kingpin.ParseContext) error { @@ -29,6 +31,7 @@ func main() { kpApp.Command("start", "Start events processing."). Default(). Action(func(c *kingpin.ParseContext) error { + shell_operator_app.SetupLogging() // Be a good parent - clean up after the child processes // in case if addon-operator is a PID 1 process. go executor.Reap() diff --git a/go.mod b/go.mod index 858a0598..2b3f769d 100644 --- a/go.mod +++ b/go.mod @@ -4,15 +4,14 @@ go 1.12 require ( github.com/evanphx/json-patch v4.5.0+incompatible - github.com/flant/shell-operator v1.0.0-beta.5.0.20191008212016-a10d5213a3fe // branch: master + github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71 // branch: json_logging github.com/go-openapi/spec v0.19.3 github.com/kennygrant/sanitize v1.2.4 github.com/otiai10/copy v1.0.1 github.com/peterbourgon/mergemap v0.0.0-20130613134717-e21c03b7a721 github.com/prometheus/client_golang v1.0.0 - github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734 - github.com/sirupsen/logrus v1.2.0 + github.com/sirupsen/logrus v1.4.2 github.com/stretchr/testify v1.4.0 golang.org/x/tools v0.0.0-20190627033414-4874f863e654 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 diff --git a/go.sum b/go.sum index 307031d1..4463e5e3 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5I github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/flant/go-openapi-validate v0.19.4-0.20190926112101-38fbca4ac77f h1:HTLgtnIbx2CVK74aTk1D4XbOh6tOHsPShw7UPOJTQzM= github.com/flant/go-openapi-validate v0.19.4-0.20190926112101-38fbca4ac77f/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= -github.com/flant/shell-operator v1.0.0-beta.5.0.20191008212016-a10d5213a3fe h1:M+aKrwzUMMxroqp14zUf+EEA9+AH2BiCqtkJ1Y7cGoE= -github.com/flant/shell-operator v1.0.0-beta.5.0.20191008212016-a10d5213a3fe/go.mod h1:rJObYHs157lh7yXCHfzakhEQlWLyqlppLBLq8qbHORQ= +github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71 h1:qjHc7vp7uBCpMbenhkgSs2uqVkZ/oLrT3uBxD+4Xnhc= +github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71/go.mod h1:KQ6PubnYxbkW8BoAXfWVra5dGuMSrec/G4ztFQEC2jU= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -175,12 +175,11 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI= -github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0= github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734 h1:Cpx2WLIv6fuPvaJAHNhYOgYzk/8RcJXu/8+mOrxf2KM= github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734/go.mod h1:hqVOMAwu+ekffC3Tvq5N1ljnXRrFKcaSjbCmQ8JgYaI= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -238,6 +237,7 @@ golang.org/x/sys v0.0.0-20190312061237-fead79001313 h1:pczuHS43Cp2ktBEEmLwScxgjW golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pkg/addon-operator/operator.go b/pkg/addon-operator/operator.go index 1c352688..d8fe6967 100644 --- a/pkg/addon-operator/operator.go +++ b/pkg/addon-operator/operator.go @@ -11,10 +11,10 @@ import ( "path" "time" - "github.com/flant/shell-operator/pkg/hook" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" + "github.com/flant/shell-operator/pkg/hook" "github.com/flant/shell-operator/pkg/kube" "github.com/flant/shell-operator/pkg/kube_events_manager" "github.com/flant/shell-operator/pkg/metrics_storage" @@ -73,13 +73,13 @@ var ( // // Creating an empty queue with jobs. func Init() error { - rlog.Debug("INIT: started") + log.Debug("INIT: started") var err error cwd, err := os.Getwd() if err != nil { - rlog.Errorf("INIT: Cannot get current working directory of process: %s", err) + log.Errorf("INIT: Cannot get current working directory of process: %s", err) return err } @@ -92,15 +92,15 @@ func Init() error { if GlobalHooksDir == "" { GlobalHooksDir = path.Join(cwd, app.GlobalHooksDir) } - rlog.Infof("INIT: Modules: '%s', Global hooks: '%s'", ModulesDir, GlobalHooksDir) + log.Infof("INIT: Modules: '%s', Global hooks: '%s'", ModulesDir, GlobalHooksDir) TempDir := app.TmpDir err = os.MkdirAll(TempDir, os.FileMode(0777)) if err != nil { - rlog.Errorf("INIT: Cannot create temporary dir '%s': %s", TempDir, err) + log.Errorf("INIT: Cannot create temporary dir '%s': %s", TempDir, err) return err } - rlog.Infof("INIT: Temporary dir: %s", TempDir) + log.Infof("INIT: Temporary dir: %s", TempDir) // init and start metrics gathering loop @@ -111,7 +111,7 @@ func Init() error { // Initializing the empty task queue and queue dumper TasksQueue = task.NewTasksQueue() // Initializing the queue dumper, which writes queue changes to the dump file. - rlog.Debugf("INIT: Tasks queue dump file: '%s'", app.TasksQueueDumpFilePath) + log.Debugf("INIT: Tasks queue dump file: '%s'", app.TasksQueueDumpFilePath) queueWatcher := task.NewTasksQueueDumper(app.TasksQueueDumpFilePath, TasksQueue) TasksQueue.AddWatcher(queueWatcher) @@ -119,14 +119,14 @@ func Init() error { // Initializing the connection to the k8s. err = kube.Init(kube.InitOptions{}) if err != nil { - rlog.Errorf("INIT: Cannot initialize Kubernetes client: %s", err) + log.Errorf("INIT: Cannot initialize Kubernetes client: %s", err) return err } // A useful callback when addon-operator is used as library if BeforeHelmInitCb != nil { - rlog.Debugf("INIT: run BeforeHelmInitCallback") + log.Debugf("INIT: run BeforeHelmInitCallback") BeforeHelmInitCb() } @@ -139,14 +139,14 @@ func Init() error { ProbeListenPort: app.TillerProbeListenPort, }) if err != nil { - rlog.Errorf("INIT: Tiller is failed to start: %s", err) + log.Errorf("INIT: Tiller is failed to start: %s", err) return err } // Initializing helm client err = helm.InitClient() if err != nil { - rlog.Errorf("INIT: helm client: %s", err) + log.Errorf("INIT: helm client: %s", err) return err } @@ -168,7 +168,7 @@ func Init() error { ModuleManager.WithKubeConfigManager(KubeConfigManager) err = ModuleManager.Init() if err != nil { - rlog.Errorf("INIT: Cannot initialize module manager: %s", err) + log.Errorf("INIT: Cannot initialize module manager: %s", err) return err } @@ -176,7 +176,7 @@ func Init() error { // Initializing the hooks schedule. ScheduleManager, err = schedule_manager.Init() if err != nil { - rlog.Errorf("INIT: Cannot initialize schedule manager: %s", err) + log.Errorf("INIT: Cannot initialize schedule manager: %s", err) return err } @@ -195,7 +195,7 @@ func Init() error { //// Initialize kube events //KubeEventsManager, err = kube_events_manager.Init() //if err != nil { - // rlog.Errorf("INIT: Cannot initialize kube events manager: %s", err) + // log.Errorf("INIT: Cannot initialize kube events manager: %s", err) // return err //} //KubernetesHooksController = kube_event_hook.NewKubernetesHooksController() @@ -209,7 +209,7 @@ func Init() error { func Run() { // Loading the onStartup hooks into the queue and running all modules. // Turning tracking changes on only after startup ends. - rlog.Info("MAIN: Start. Trigger onStartup event.") + log.Info("MAIN: Start. Trigger onStartup event.") TasksQueue.ChangesDisable() CreateOnStartupTasks() @@ -218,7 +218,7 @@ func Run() { err := KubernetesHooksController.EnableGlobalHooks() if err != nil { // Something wrong with global hook configs, cannot start informers. - rlog.Errorf("Start informers for global kubernetes hooks: %v", err) + log.Errorf("Start informers for global kubernetes hooks: %v", err) return } // Start all created informers @@ -251,23 +251,23 @@ func ManagersEventsHandler() { // Some modules have changed. case module_manager.ModulesChanged: for _, moduleChange := range moduleEvent.ModulesChanges { - rlog.Infof("EVENT ModulesChanged, type=Changed") + log.Infof("EVENT ModulesChanged, type=Changed") newTask := task.NewTask(task.ModuleRun, moduleChange.Name) TasksQueue.Add(newTask) - rlog.Infof("QUEUE add ModuleRun %s", newTask.Name) + log.Infof("QUEUE add ModuleRun %s", newTask.Name) } // As module list may have changed, hook schedule index must be re-created. ScheduleHooksController.UpdateScheduleHooks() case module_manager.GlobalChanged: // Global values are changed, all modules must be restarted. - rlog.Infof("EVENT GlobalChanged") + log.Infof("EVENT GlobalChanged") TasksQueue.ChangesDisable() CreateReloadAllTasks(false) TasksQueue.ChangesEnable(true) // As module list may have changed, hook schedule index must be re-created. ScheduleHooksController.UpdateScheduleHooks() case module_manager.AmbigousState: - rlog.Infof("EVENT AmbiguousState") + log.Infof("EVENT AmbiguousState") TasksQueue.ChangesDisable() // It is the error in the module manager. The task must be added to // the beginning of the queue so the module manager can restore its @@ -277,37 +277,37 @@ func ManagersEventsHandler() { // It is the delay before retry. TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) TasksQueue.ChangesEnable(true) - rlog.Infof("QUEUE push ModuleManagerRetry, push FailedModuleDelay") + log.Infof("QUEUE push ModuleManagerRetry, push FailedModuleDelay") } case crontab := <-schedule_manager.ScheduleCh: - rlog.Infof("EVENT Schedule event '%s'", crontab) + log.Infof("EVENT Schedule event '%s'", crontab) tasks, err := ScheduleHooksController.HandleEvent(crontab) if err != nil { - rlog.Errorf("MAIN_LOOP Schedule event '%s': %s", crontab, err) + log.Errorf("MAIN_LOOP Schedule event '%s': %s", crontab, err) break } for _, resTask := range tasks { TasksQueue.Add(resTask) - rlog.Infof("QUEUE add %s@%s %s", resTask.GetType(), resTask.GetBinding(), resTask.GetName()) + log.Infof("QUEUE add %s@%s %s", resTask.GetType(), resTask.GetBinding(), resTask.GetName()) } case kubeEvent := <-kube_events_manager.KubeEventCh: - rlog.Infof("EVENT Kube event '%s'", kubeEvent.ConfigId) + log.Infof("EVENT Kube event '%s'", kubeEvent.ConfigId) tasks, err := KubernetesHooksController.HandleEvent(kubeEvent) if err != nil { - rlog.Errorf("MAIN_LOOP error handling kube event '%s': %s", kubeEvent.ConfigId, err) + log.Errorf("MAIN_LOOP error handling kube event '%s': %s", kubeEvent.ConfigId, err) break } for _, t := range tasks { TasksQueue.Add(t) - rlog.Infof("QUEUE add %s@%s %s", t.GetType(), t.GetBinding(), t.GetName()) + log.Infof("QUEUE add %s@%s %s", t.GetType(), t.GetBinding(), t.GetName()) } case <-ManagersEventsHandlerStopCh: - rlog.Infof("EVENT Stop") + log.Infof("EVENT Stop") return } } @@ -333,7 +333,7 @@ func runDiscoverModulesState(discoverTask task.Task) error { WithOnStartupHooks(runOnStartupHooks) TasksQueue.Add(newTask) - rlog.Infof("QUEUE add ModuleRun %s", moduleName) + log.Infof("QUEUE add ModuleRun %s", moduleName) } for _, moduleName := range modulesState.ModulesToDisable { @@ -348,13 +348,13 @@ func runDiscoverModulesState(discoverTask task.Task) error { } newTask := task.NewTask(task.ModuleDelete, moduleName) TasksQueue.Add(newTask) - rlog.Infof("QUEUE add ModuleDelete %s", moduleName) + log.Infof("QUEUE add ModuleDelete %s", moduleName) } for _, moduleName := range modulesState.ReleasedUnknownModules { newTask := task.NewTask(task.ModulePurge, moduleName) TasksQueue.Add(newTask) - rlog.Infof("QUEUE add ModulePurge %s", moduleName) + log.Infof("QUEUE add ModulePurge %s", moduleName) } // Queue afterAll global hooks @@ -364,10 +364,10 @@ func runDiscoverModulesState(discoverTask task.Task) error { WithBinding(module_manager.AfterAll). AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: module_manager.ContextBindingType[module_manager.AfterAll]}}) TasksQueue.Add(newTask) - rlog.Debugf("QUEUE add GlobalHookRun@AfterAll '%s'", hookName) + log.Debugf("QUEUE add GlobalHookRun@AfterAll '%s'", hookName) } if len(afterAllHooks) > 0 { - rlog.Infof("QUEUE add all GlobalHookRun@AfterAll") + log.Infof("QUEUE add all GlobalHookRun@AfterAll") } ScheduleHooksController.UpdateScheduleHooks() @@ -410,45 +410,45 @@ func TasksRunner() { switch t.GetType() { case task.DiscoverModulesState: - rlog.Infof("TASK_RUN DiscoverModulesState") + log.Infof("TASK_RUN DiscoverModulesState") err := runDiscoverModulesState(t) if err != nil { MetricsStorage.SendCounterMetric(PrefixMetric("modules_discover_errors"), 1.0, map[string]string{}) t.IncrementFailureCount() - rlog.Errorf("TASK_RUN %s failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetFailureCount(), err) + log.Errorf("TASK_RUN %s failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetFailureCount(), err) TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) - rlog.Infof("QUEUE push FailedModuleDelay") + log.Infof("QUEUE push FailedModuleDelay") break } TasksQueue.Pop() case task.ModuleRun: - rlog.Infof("TASK_RUN ModuleRun %s", t.GetName()) + log.Infof("TASK_RUN ModuleRun %s", t.GetName()) err := ModuleManager.RunModule(t.GetName(), t.GetOnStartupHooks()) if err != nil { MetricsStorage.SendCounterMetric(PrefixMetric("module_run_errors"), 1.0, map[string]string{"module": t.GetName()}) t.IncrementFailureCount() - rlog.Errorf("TASK_RUN ModuleRun '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetName(), t.GetFailureCount(), err) + log.Errorf("TASK_RUN ModuleRun '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetName(), t.GetFailureCount(), err) TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) - rlog.Infof("QUEUE push FailedModuleDelay") + log.Infof("QUEUE push FailedModuleDelay") } else { TasksQueue.Pop() } case task.ModuleDelete: - rlog.Infof("TASK_RUN ModuleDelete %s", t.GetName()) + log.Infof("TASK_RUN ModuleDelete %s", t.GetName()) err := ModuleManager.DeleteModule(t.GetName()) if err != nil { MetricsStorage.SendCounterMetric(PrefixMetric("module_delete_errors"), 1.0, map[string]string{"module": t.GetName()}) t.IncrementFailureCount() - rlog.Errorf("%s '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetName(), t.GetFailureCount(), err) + log.Errorf("%s '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetName(), t.GetFailureCount(), err) TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) - rlog.Infof("QUEUE push FailedModuleDelay") + log.Infof("QUEUE push FailedModuleDelay") } else { TasksQueue.Pop() } case task.ModuleHookRun: - rlog.Infof("TASK_RUN ModuleHookRun@%s %s", t.GetBinding(), t.GetName()) + log.Infof("TASK_RUN ModuleHookRun@%s %s", t.GetBinding(), t.GetName()) err := ModuleManager.RunModuleHook(t.GetName(), t.GetBinding(), t.GetBindingContext()) if err != nil { moduleHook, _ := ModuleManager.GetModuleHook(t.GetName()) @@ -461,15 +461,15 @@ func TasksRunner() { } else { MetricsStorage.SendCounterMetric(PrefixMetric("module_hook_errors"), 1.0, map[string]string{"module": moduleLabel, "hook": hookLabel}) t.IncrementFailureCount() - rlog.Errorf("%s '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetName(), t.GetFailureCount(), err) + log.Errorf("%s '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetName(), t.GetFailureCount(), err) TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) - rlog.Infof("QUEUE push FailedModuleDelay") + log.Infof("QUEUE push FailedModuleDelay") } } else { TasksQueue.Pop() } case task.GlobalHookRun: - rlog.Infof("TASK_RUN GlobalHookRun@%s %s", t.GetBinding(), t.GetName()) + log.Infof("TASK_RUN GlobalHookRun@%s %s", t.GetBinding(), t.GetName()) err := ModuleManager.RunGlobalHook(t.GetName(), t.GetBinding(), t.GetBindingContext()) if err != nil { globalHook, _ := ModuleManager.GetGlobalHook(t.GetName()) @@ -481,41 +481,52 @@ func TasksRunner() { } else { MetricsStorage.SendCounterMetric(PrefixMetric("global_hook_errors"), 1.0, map[string]string{"hook": hookLabel}) t.IncrementFailureCount() - rlog.Errorf("TASK_RUN %s '%s' on '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetName(), t.GetBinding(), t.GetFailureCount(), err) + log.Errorf("TASK_RUN %s '%s' on '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetName(), t.GetBinding(), t.GetFailureCount(), err) TasksQueue.Push(task.NewTaskDelay(FailedHookDelay)) } } else { TasksQueue.Pop() } case task.ModulePurge: - rlog.Infof("TASK_RUN ModulePurge %s", t.GetName()) + log. + WithField("operator.component", "taskRunner"). + WithField("task", "ModulePurge"). + WithField("module", t.GetName()). + Debugf("run task") + logEntry := log.WithField("module", t.GetName()).WithField("phase", "purge") // Module for purge is unknown so log deletion error is enough. - err := helm.Client.DeleteRelease(t.GetName()) + err := helm.NewHelmCli(logEntry).DeleteRelease(t.GetName()) if err != nil { - rlog.Errorf("TASK_RUN %s Helm delete '%s' failed. Error: %s", t.GetType(), t.GetName(), err) + log.Errorf("TASK_RUN %s Helm delete '%s' failed. Error: %s", t.GetType(), t.GetName(), err) } TasksQueue.Pop() case task.ModuleManagerRetry: - rlog.Infof("TASK_RUN ModuleManagerRetry") + log. + WithField("operator.component", "taskRunner"). + WithField("task", "ModuleManagerRetry"). + WithField("module", t.GetName()). + Infof("Retry") MetricsStorage.SendCounterMetric(PrefixMetric("modules_discover_errors"), 1.0, map[string]string{}) ModuleManager.Retry() TasksQueue.Pop() // Adding a delay before retrying module/hook task. - TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) - rlog.Infof("QUEUE push FailedModuleDelay") + newTask := task.NewTaskDelay(FailedModuleDelay) + newTask.Name = t.GetName() + TasksQueue.Push(newTask) + log.Infof("QUEUE push FailedModuleDelay") case task.Delay: - rlog.Infof("TASK_RUN Delay for %s", t.GetDelay().String()) + log.Infof("TASK_RUN Delay for %s", t.GetDelay().String()) TasksQueue.Pop() time.Sleep(t.GetDelay()) case task.Stop: - rlog.Infof("TASK_RUN Stop: Exiting TASK_RUN loop.") + log.Infof("TASK_RUN Stop: Exiting TASK_RUN loop.") TasksQueue.Pop() return } // Breaking, if the task queue is empty to prevent the infinite loop. if TasksQueue.IsEmpty() { - rlog.Debug("Task queue is empty. Will sleep now.") + log.Debug("Task queue is empty. Will sleep now.") break } } @@ -523,7 +534,7 @@ func TasksRunner() { } func CreateOnStartupTasks() { - rlog.Infof("QUEUE add all GlobalHookRun@OnStartup") + log.Infof("QUEUE add all GlobalHookRun@OnStartup") onStartupHooks := ModuleManager.GetGlobalHooksInOrder(module_manager.OnStartup) @@ -532,14 +543,14 @@ func CreateOnStartupTasks() { WithBinding(module_manager.OnStartup). AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: module_manager.ContextBindingType[module_manager.OnStartup]}}) TasksQueue.Add(newTask) - rlog.Debugf("QUEUE add GlobalHookRun@OnStartup '%s'", hookName) + log.Debugf("QUEUE add GlobalHookRun@OnStartup '%s'", hookName) } return } func CreateReloadAllTasks(onStartup bool) { - rlog.Infof("QUEUE add all GlobalHookRun@BeforeAll, add DiscoverModulesState") + log.Infof("QUEUE add all GlobalHookRun@BeforeAll, add DiscoverModulesState") // Queue beforeAll global hooks. beforeAllHooks := ModuleManager.GetGlobalHooksInOrder(module_manager.BeforeAll) @@ -550,7 +561,7 @@ func CreateReloadAllTasks(onStartup bool) { AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: module_manager.ContextBindingType[module_manager.BeforeAll]}}) TasksQueue.Add(newTask) - rlog.Debugf("QUEUE GlobalHookRun@BeforeAll '%s'", module_manager.BeforeAll, hookName) + log.Debugf("QUEUE GlobalHookRun@BeforeAll '%s'", module_manager.BeforeAll, hookName) } TasksQueue.Add(task.NewTask(task.DiscoverModulesState, "").WithOnStartupHooks(onStartup)) @@ -595,7 +606,7 @@ func InitHttpServer(listenAddr string, listenPort string) error { }) address := fmt.Sprintf("%s:%s", listenAddr, listenPort) - rlog.Infof("HTTP SERVER Listening on %s", address) + log.Infof("HTTP SERVER Listening on %s", address) // Check if port is available ln, err := net.Listen("tcp", address) @@ -605,7 +616,7 @@ func InitHttpServer(listenAddr string, listenPort string) error { go func() { if err := http.Serve(ln, nil); err != nil { - rlog.Errorf("Error starting HTTP server: %s", err) + log.Errorf("Error starting HTTP server: %s", err) os.Exit(1) } }() diff --git a/pkg/addon-operator/start.go b/pkg/addon-operator/start.go index 9ed68f46..4ef21cc7 100644 --- a/pkg/addon-operator/start.go +++ b/pkg/addon-operator/start.go @@ -3,7 +3,7 @@ package addon_operator import ( "os" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" shell_operator_app "github.com/flant/shell-operator/pkg/app" @@ -16,18 +16,17 @@ func Start() { err = InitHttpServer(app.ListenAddress, app.ListenPort) if err != nil { - rlog.Errorf("HTTP SERVER start failed: %v", err) + log.Errorf("HTTP SERVER start failed: %v", err) os.Exit(1) } - rlog.Infof("addon-operator %s, shell-operator %s", app.Version, shell_operator_app.Version) + log.Infof("addon-operator %s, shell-operator %s", app.Version, shell_operator_app.Version) err = Init() if err != nil { - rlog.Errorf("INIT failed: %v", err) + log.Errorf("INIT failed: %v", err) os.Exit(1) } - rlog.Debugf("START: Run") Run() } diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 42f9cdb2..6b353901 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -10,7 +10,7 @@ import ( "strconv" "strings" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kblabels "k8s.io/apimachinery/pkg/labels" @@ -35,37 +35,47 @@ type HelmClient interface { ListReleases(labelSelector map[string]string) ([]string, error) ListReleasesNames(labelSelector map[string]string) ([]string, error) IsReleaseExists(releaseName string) (bool, error) + WithLog(entry *log.Entry) } var Client HelmClient type CliHelm struct { + LogEntry *log.Entry } -// Init starts Tiller installation. +var _ HelmClient = &CliHelm{} + +// InitClient initialize helm client func InitClient() error { - rlog.Info("Helm: run helm init") + log.Info("Helm: run helm init") cliHelm := &CliHelm{} - // initialize helm client stdout, stderr, err := cliHelm.Cmd("init", "--client-only") if err != nil { return fmt.Errorf("helm init: %v\n%v %v", err, stdout, stderr) } - - stdout, stderr, err = cliHelm.Cmd("version") + stdout, stderr, err = cliHelm.Cmd("version", "--short") if err != nil { - return fmt.Errorf("unable to get helm version: %v\n%v %v", err, stdout, stderr) + return fmt.Errorf("unable to get helm or tiller version: %v\n%v %v", err, stdout, stderr) } - rlog.Infof("Helm: helm version:\n%v %v", stdout, stderr) + stdout = strings.Join([]string{stdout, stderr}, "\n") + stdout = strings.ReplaceAll(stdout, "\n", " ") + log.Infof("Helm: successfully initialized. Version: %s", stdout) - rlog.Info("Helm: successfully initialized") + return nil +} - Client = cliHelm +var NewHelmCli = func(logEntry *log.Entry) HelmClient { + return &CliHelm{ + LogEntry: logEntry.WithField("operator.component", "helm"), + } +} - return nil +func (h *CliHelm) WithLog(logEntry *log.Entry) { + h.LogEntry = logEntry.WithField("operator.component", "helm") } func (h *CliHelm) TillerNamespace() string { @@ -82,9 +92,9 @@ func (h *CliHelm) CommandEnv() []string { // Cmd starts Helm with specified arguments. // Sets the TILLER_NAMESPACE environment variable before starting, because Addon-operator works with its own Tiller. -func (helm *CliHelm) Cmd(args ...string) (stdout string, stderr string, err error) { +func (h *CliHelm) Cmd(args ...string) (stdout string, stderr string, err error) { cmd := exec.Command(HelmPath, args...) - cmd.Env = append(os.Environ(), helm.CommandEnv()...) + cmd.Env = append(os.Environ(), h.CommandEnv()...) var stdoutBuf bytes.Buffer cmd.Stdout = &stdoutBuf @@ -98,41 +108,41 @@ func (helm *CliHelm) Cmd(args ...string) (stdout string, stderr string, err erro return } -func (helm *CliHelm) DeleteSingleFailedRevision(releaseName string) (err error) { - revision, status, err := helm.LastReleaseStatus(releaseName) +func (h *CliHelm) DeleteSingleFailedRevision(releaseName string) (err error) { + revision, status, err := h.LastReleaseStatus(releaseName) if err != nil { if revision == "0" { // Revision 0 is not an error. Just skips deletion. - rlog.Debugf("helm release '%s': Release not found, no cleanup required.", releaseName) + log.Debugf("helm release '%s': Release not found, no cleanup required.", releaseName) return nil } - rlog.Errorf("helm release '%s': got error from LastReleaseStatus: %s", releaseName, err) + h.LogEntry.Errorf("helm release '%s': got error from LastReleaseStatus: %s", releaseName, err) return err } if revision == "1" && status == "FAILED" { // Deletes and purges! - err = helm.DeleteRelease(releaseName) + err = h.DeleteRelease(releaseName) if err != nil { - rlog.Errorf("helm release '%s': cleanup of failed revision got error: %v", releaseName, err) + h.LogEntry.Errorf("helm release '%s': cleanup of failed revision got error: %v", releaseName, err) return err } - rlog.Infof("helm release '%s': cleanup of failed revision succeeded", releaseName) + h.LogEntry.Infof("helm release '%s': cleanup of failed revision succeeded", releaseName) } else { // No interest of revisions older than 1. - rlog.Debugf("helm release '%s': has revision '%s' with status %s", releaseName, revision, status) + h.LogEntry.Debugf("helm release '%s': has revision '%s' with status %s", releaseName, revision, status) } return } -func (helm *CliHelm) DeleteOldFailedRevisions(releaseName string) error { - cmNames, err := helm.ListReleases(map[string]string{"STATUS": "FAILED", "NAME": releaseName}) +func (h *CliHelm) DeleteOldFailedRevisions(releaseName string) error { + cmNames, err := h.ListReleases(map[string]string{"STATUS": "FAILED", "NAME": releaseName}) if err != nil { return err } - rlog.Debugf("helm release '%s': found ConfigMaps: %v", cmNames) + h.LogEntry.Debugf("helm release '%s': found ConfigMaps: %v", releaseName, cmNames) var releaseCmNamePattern = regexp.MustCompile(`^(.*).v([0-9]+)$`) @@ -156,7 +166,7 @@ func (helm *CliHelm) DeleteOldFailedRevisions(releaseName string) error { for _, revision := range revisions { cmName := fmt.Sprintf("%s.v%d", releaseName, revision) - rlog.Infof("helm release '%s': delete old FAILED revision cm/%s", releaseName, cmName) + h.LogEntry.Infof("helm release '%s': delete old FAILED revision cm/%s", releaseName, cmName) err := kube.Kubernetes.CoreV1(). ConfigMaps(app.Namespace). @@ -175,8 +185,8 @@ func (helm *CliHelm) DeleteOldFailedRevisions(releaseName string) error { // helm history output: // REVISION UPDATED STATUS CHART DESCRIPTION // 1 Fri Jul 14 18:25:00 2017 SUPERSEDED symfony-demo-0.1.0 Install complete -func (helm *CliHelm) LastReleaseStatus(releaseName string) (revision string, status string, err error) { - stdout, stderr, err := helm.Cmd("history", releaseName, "--max", "1") +func (h *CliHelm) LastReleaseStatus(releaseName string) (revision string, status string, err error) { + stdout, stderr, err := h.Cmd("history", releaseName, "--max", "1") if err != nil { errLine := strings.Split(stderr, "\n")[0] @@ -199,7 +209,7 @@ func (helm *CliHelm) LastReleaseStatus(releaseName string) (revision string, sta return } -func (helm *CliHelm) UpgradeRelease(releaseName string, chart string, valuesPaths []string, setValues []string, namespace string) error { +func (h *CliHelm) UpgradeRelease(releaseName string, chart string, valuesPaths []string, setValues []string, namespace string) error { args := make([]string, 0) args = append(args, "upgrade") args = append(args, "--install") @@ -221,18 +231,18 @@ func (helm *CliHelm) UpgradeRelease(releaseName string, chart string, valuesPath args = append(args, setValue) } - rlog.Infof("Running helm upgrade for release '%s' with chart '%s' in namespace '%s' ...", releaseName, chart, namespace) - stdout, stderr, err := helm.Cmd(args...) + h.LogEntry.Infof("Running helm upgrade for release '%s' with chart '%s' in namespace '%s' ...", releaseName, chart, namespace) + stdout, stderr, err := h.Cmd(args...) if err != nil { return fmt.Errorf("helm upgrade failed: %s:\n%s %s", err, stdout, stderr) } - rlog.Infof("Helm upgrade for release '%s' with chart '%s' in namespace '%s' successful:\n%s\n%s", releaseName, chart, namespace, stdout, stderr) + h.LogEntry.Infof("Helm upgrade for release '%s' with chart '%s' in namespace '%s' successful:\n%s\n%s", releaseName, chart, namespace, stdout, stderr) return nil } -func (helm *CliHelm) GetReleaseValues(releaseName string) (utils.Values, error) { - stdout, stderr, err := helm.Cmd("get", "values", releaseName) +func (h *CliHelm) GetReleaseValues(releaseName string) (utils.Values, error) { + stdout, stderr, err := h.Cmd("get", "values", releaseName) if err != nil { return nil, fmt.Errorf("cannot get values of helm release %s: %s\n%s %s", releaseName, err, stdout, stderr) } @@ -245,10 +255,10 @@ func (helm *CliHelm) GetReleaseValues(releaseName string) (utils.Values, error) return values, nil } -func (helm *CliHelm) DeleteRelease(releaseName string) (err error) { - rlog.Debugf("helm release '%s': execute helm delete --purge", releaseName) +func (h *CliHelm) DeleteRelease(releaseName string) (err error) { + h.LogEntry.Debugf("helm release '%s': execute helm delete --purge", releaseName) - stdout, stderr, err := helm.Cmd("delete", "--purge", releaseName) + stdout, stderr, err := h.Cmd("delete", "--purge", releaseName) if err != nil { return fmt.Errorf("helm delete --purge %s invocation error: %v\n%v %v", releaseName, err, stdout, stderr) } @@ -256,8 +266,8 @@ func (helm *CliHelm) DeleteRelease(releaseName string) (err error) { return } -func (helm *CliHelm) IsReleaseExists(releaseName string) (bool, error) { - revision, _, err := helm.LastReleaseStatus(releaseName) +func (h *CliHelm) IsReleaseExists(releaseName string) (bool, error) { + revision, _, err := h.LastReleaseStatus(releaseName) if err != nil && revision == "0" { return false, nil } else if err != nil { @@ -270,7 +280,7 @@ func (helm *CliHelm) IsReleaseExists(releaseName string) (bool, error) { // Returns all known releases as strings — ".v" // Helm looks for ConfigMaps by label 'OWNER=TILLER' and gets release info from the 'release' key. // https://github.com/kubernetes/helm/blob/8981575082ea6fc2a670f81fb6ca5b560c4f36a7/pkg/storage/driver/cfgmaps.go#L88 -func (helm *CliHelm) ListReleases(labelSelector map[string]string) ([]string, error) { +func (h *CliHelm) ListReleases(labelSelector map[string]string) ([]string, error) { labelsSet := make(kblabels.Set) for k, v := range labelSelector { labelsSet[k] = v @@ -281,7 +291,7 @@ func (helm *CliHelm) ListReleases(labelSelector map[string]string) ([]string, er ConfigMaps(app.Namespace). List(metav1.ListOptions{LabelSelector: labelsSet.AsSelector().String()}) if err != nil { - rlog.Debugf("helm: list of releases ConfigMaps failed: %s", err) + h.LogEntry.Debugf("helm: list of releases ConfigMaps failed: %s", err) return nil, err } @@ -298,8 +308,8 @@ func (helm *CliHelm) ListReleases(labelSelector map[string]string) ([]string, er } // ListReleasesNames returns list of release names without suffixes ".v" -func (helm *CliHelm) ListReleasesNames(labelSelector map[string]string) ([]string, error) { - releases, err := helm.ListReleases(labelSelector) +func (h *CliHelm) ListReleasesNames(labelSelector map[string]string) ([]string, error) { + releases, err := h.ListReleases(labelSelector) if err != nil { return []string{}, err } diff --git a/pkg/helm/helm_test.go b/pkg/helm/helm_test.go index 59f395b8..917b5005 100644 --- a/pkg/helm/helm_test.go +++ b/pkg/helm/helm_test.go @@ -8,12 +8,34 @@ import ( "sort" "testing" + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" "k8s.io/api/rbac/v1beta1" "github.com/flant/shell-operator/pkg/kube" ) +func Test_Logging(t *testing.T) { + log.Info("Start test") + + logEntry1 := log.WithField("test", "helm") + logEntry1.Infof("asd") + + logEntry2 := log.WithField("test2", "helm2") + logEntry2.Infof("asd") + + logEntry11 := logEntry1.WithField("subtest", "helmm") + logEntry11.Infof("helmm info") + + logEntry1.Infof("asd again") + + + logEntry11.WithField("test","helm11").Infof("helmm info") + + +} + func getTestDirectoryPath(testName string) string { _, testFile, _, _ := runtime.Caller(0) return filepath.Join(filepath.Dir(testFile), "testdata", testName) @@ -77,7 +99,6 @@ func TestHelm(t *testing.T) { var releases []string helm := &CliHelm{} - //rlog.Infof("Testing tiller in '%s' namespace", helm.TillerNamespace()) _ = kube.Init(kube.InitOptions{}) //kube.AddonOperatorNamespace = helm.TillerNamespace() diff --git a/pkg/helm/tiller.go b/pkg/helm/tiller.go index beb50354..6b609423 100644 --- a/pkg/helm/tiller.go +++ b/pkg/helm/tiller.go @@ -8,7 +8,7 @@ import ( "os/exec" "time" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" ) const TillerPath = "tiller" @@ -43,7 +43,7 @@ func InitTillerProcess(options TillerOptions) error { err := tillerCmd.Start() if err != nil { - rlog.Errorf("Tiller process not started: %v", err) + log.Errorf("Tiller process not started: %v", err) return err } @@ -51,12 +51,12 @@ func InitTillerProcess(options TillerOptions) error { for { cliHelm := &CliHelm{} stdout, stderr, err := cliHelm.Cmd("version") - rlog.Debugf("helm version: %s %s", stdout, stderr) + log.Debugf("helm version: %s %s", stdout, stderr) if err != nil { - rlog.Errorf("unable to get helm version: %v\n%v %v", err, stdout, stderr) + log.Errorf("unable to get helm version: %v\n%v %v", err, stdout, stderr) time.Sleep(100*time.Millisecond) } else { - rlog.Infof("tiller started and is available") + log.Infof("tiller started and is available") break } } @@ -64,9 +64,9 @@ func InitTillerProcess(options TillerOptions) error { go func() { err = tillerCmd.Wait() if err != nil { - rlog.Errorf("Tiller process exited, now stop. (%v)", err) + log.Errorf("Tiller process exited, now stop. (%v)", err) } else { - rlog.Errorf("Tiller process exited, now stop.") + log.Errorf("Tiller process exited, now stop.") } os.Exit(1) }() diff --git a/pkg/kube_config_manager/kube_config_manager.go b/pkg/kube_config_manager/kube_config_manager.go index 0ee39a5b..4ccee2d4 100644 --- a/pkg/kube_config_manager/kube_config_manager.go +++ b/pkg/kube_config_manager/kube_config_manager.go @@ -6,7 +6,7 @@ import ( "os" "time" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" "k8s.io/api/core/v1" @@ -166,7 +166,7 @@ func (kcm *kubeConfigManager) SetKubeGlobalValues(values utils.Values) error { globalKubeConfig := GetGlobalKubeConfigFromValues(values) if globalKubeConfig != nil { - rlog.Debugf("Kube config manager: set kube global values:\n%s", utils.ValuesToString(values)) + log.Debugf("Kube config manager: set kube global values:\n%s", utils.ValuesToString(values)) err := kcm.saveGlobalKubeConfig(*globalKubeConfig) if err != nil { @@ -181,7 +181,7 @@ func (kcm *kubeConfigManager) SetKubeModuleValues(moduleName string, values util moduleKubeConfig := GetModuleKubeConfigFromValues(moduleName, values) if moduleKubeConfig != nil { - rlog.Debugf("Kube config manager: set kube module values:\n%s", moduleKubeConfig.ModuleConfig.String()) + log.Debugf("Kube config manager: set kube module values:\n%s", moduleKubeConfig.ModuleConfig.String()) err := kcm.saveModuleKubeConfig(*moduleKubeConfig) if err != nil { @@ -214,10 +214,10 @@ func (kcm *kubeConfigManager) getConfigMap() (*v1.ConfigMap, error) { if err != nil { return nil, err } - rlog.Debugf("KUBE_CONFIG_MANAGER: Will use ConfigMap/%s for persistent values", kcm.ConfigMapName) + log.Debugf("KUBE_CONFIG_MANAGER: Will use ConfigMap/%s for persistent values", kcm.ConfigMapName) return obj, nil } else { - rlog.Debugf("KUBE_CONFIG_MANAGER: ConfigMap/%s is not created", kcm.ConfigMapName) + log.Debugf("KUBE_CONFIG_MANAGER: ConfigMap/%s is not created", kcm.ConfigMapName) return nil, nil } } @@ -239,7 +239,7 @@ func (kcm *kubeConfigManager) initConfig() error { } if obj == nil { - rlog.Infof("Init config from ConfigMap: cm/%s is not found", kcm.ConfigMapName) + log.Infof("Init config from ConfigMap: cm/%s is not found", kcm.ConfigMapName) return nil } @@ -275,7 +275,7 @@ func (kcm *kubeConfigManager) initConfig() error { } func (kcm *kubeConfigManager) Init() error { - rlog.Debug("INIT: KUBE_CONFIG") + log.Debug("INIT: KUBE_CONFIG") VerboseDebug = false if os.Getenv("KUBE_CONFIG_MANAGER_DEBUG") != "" { @@ -345,7 +345,7 @@ func (kcm *kubeConfigManager) handleNewCm(obj *v1.ConfigMap) error { isGlobalDeleted := globalKubeConfig == nil && kcm.GlobalValuesChecksum != "" if isGlobalUpdated || isGlobalDeleted { - rlog.Infof("Kube config manager: detect changes in global section") + log.Infof("Kube config manager: detect changes in global section") newConfig := NewConfig() // calculate new checksum of a global section @@ -370,10 +370,10 @@ func (kcm *kubeConfigManager) handleNewCm(obj *v1.ConfigMap) error { } kcm.ModulesValuesChecksum = newModulesValuesChecksum - rlog.Debugf("Kube config manager: global section new values:\n%s", + log.Debugf("Kube config manager: global section new values:\n%s", utils.ValuesToString(newConfig.Values)) for _, moduleConfig := range newConfig.ModuleConfigs { - rlog.Debugf("%s", moduleConfig.String()) + log.Debugf("%s", moduleConfig.String()) } ConfigUpdated <- *newConfig @@ -413,9 +413,9 @@ func (kcm *kubeConfigManager) handleNewCm(obj *v1.ConfigMap) error { } if updatedCount > 0 || removedCount > 0 { - rlog.Infof("KUBE_CONFIG Detect module sections changes: %d updated, %d removed", updatedCount, removedCount) + log.Infof("KUBE_CONFIG Detect module sections changes: %d updated, %d removed", updatedCount, removedCount) for _, moduleConfig := range moduleConfigsActual { - rlog.Debugf("%s", moduleConfig.String()) + log.Debugf("%s", moduleConfig.String()) } ModuleConfigsUpdated <- moduleConfigsActual } @@ -430,7 +430,7 @@ func (kcm *kubeConfigManager) handleCmAdd(obj *v1.ConfigMap) error { if err != nil { return err } - rlog.Debugf("Kube config manager: informer: handle ConfigMap '%s' add:\n%s", obj.Name, objYaml) + log.Debugf("Kube config manager: informer: handle ConfigMap '%s' add:\n%s", obj.Name, objYaml) } return kcm.handleNewCm(obj) @@ -442,7 +442,7 @@ func (kcm *kubeConfigManager) handleCmUpdate(_ *v1.ConfigMap, obj *v1.ConfigMap) if err != nil { return err } - rlog.Debugf("Kube config manager: informer: handle ConfigMap '%s' update:\n%s", obj.Name, objYaml) + log.Debugf("Kube config manager: informer: handle ConfigMap '%s' update:\n%s", obj.Name, objYaml) } return kcm.handleNewCm(obj) @@ -454,7 +454,7 @@ func (kcm *kubeConfigManager) handleCmDelete(obj *v1.ConfigMap) error { if err != nil { return err } - rlog.Debugf("Kube config manager: handle ConfigMap '%s' delete:\n%s", obj.Name, objYaml) + log.Debugf("Kube config manager: handle ConfigMap '%s' delete:\n%s", obj.Name, objYaml) } if kcm.GlobalValuesChecksum != "" { @@ -492,7 +492,7 @@ func (kcm *kubeConfigManager) handleCmDelete(obj *v1.ConfigMap) error { } func (kcm *kubeConfigManager) Run() { - rlog.Debugf("Run kube config manager") + log.Debugf("Run kube config manager") lw := cache.NewListWatchFromClient( kube.Kubernetes.CoreV1().RESTClient(), @@ -508,19 +508,19 @@ func (kcm *kubeConfigManager) Run() { AddFunc: func(obj interface{}) { err := kcm.handleCmAdd(obj.(*v1.ConfigMap)) if err != nil { - rlog.Errorf("Kube config manager: cannot handle ConfigMap add: %s", err) + log.Errorf("Kube config manager: cannot handle ConfigMap add: %s", err) } }, UpdateFunc: func(prevObj interface{}, obj interface{}) { err := kcm.handleCmUpdate(prevObj.(*v1.ConfigMap), obj.(*v1.ConfigMap)) if err != nil { - rlog.Errorf("Kube config manager: cannot handle ConfigMap update: %s", err) + log.Errorf("Kube config manager: cannot handle ConfigMap update: %s", err) } }, DeleteFunc: func(obj interface{}) { err := kcm.handleCmDelete(obj.(*v1.ConfigMap)) if err != nil { - rlog.Errorf("Kube config manager: cannot handle ConfigMap delete: %s", err) + log.Errorf("Kube config manager: cannot handle ConfigMap delete: %s", err) } }, }) diff --git a/pkg/kube_config_manager/module_kube_config.go b/pkg/kube_config_manager/module_kube_config.go index cc38cafc..e0adb081 100644 --- a/pkg/kube_config_manager/module_kube_config.go +++ b/pkg/kube_config_manager/module_kube_config.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" utils_checksum "github.com/flant/shell-operator/pkg/utils/checksum" @@ -13,6 +13,7 @@ import ( ) // TODO make a method of KubeConfig +// TODO LOG: multierror? // GetModulesNamesFromConfigData returns all keys in kube config except global // modNameEnabled keys are also handled func GetModulesNamesFromConfigData(configData map[string]string) map[string]bool { @@ -30,7 +31,7 @@ func GetModulesNamesFromConfigData(configData map[string]string) map[string]bool modName := utils.ModuleNameFromValuesKey(key) if utils.ModuleNameToValuesKey(modName) != key { - rlog.Errorf("Bad module name '%s': should be camelCased module name: ignoring data", key) + log.Errorf("Bad module name '%s': should be camelCased module name: ignoring data", key) continue } res[modName] = true diff --git a/pkg/module_manager/global_hook.go b/pkg/module_manager/global_hook.go index e04223eb..4e516729 100644 --- a/pkg/module_manager/global_hook.go +++ b/pkg/module_manager/global_hook.go @@ -5,7 +5,7 @@ import ( "fmt" "path/filepath" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" hook2 "github.com/flant/shell-operator/pkg/hook" utils_data "github.com/flant/shell-operator/pkg/utils/data" @@ -92,7 +92,7 @@ func (h *GlobalHook) handleGlobalValuesPatch(currentValues utils.Values, valuesP } func (h *GlobalHook) run(bindingType BindingType, context []BindingContext) error { - rlog.Infof("Running global hook '%s' binding '%s' ...", h.Name, bindingType) + log.Infof("Running global hook '%s' binding '%s' ...", h.Name, bindingType) // Convert context for version versionedContext := make([]interface{}, 0, len(context)) @@ -121,12 +121,12 @@ func (h *GlobalHook) run(bindingType BindingType, context []BindingContext) erro if configValuesPatchResult.ValuesChanged { err := h.moduleManager.kubeConfigManager.SetKubeGlobalValues(configValuesPatchResult.Values) if err != nil { - rlog.Debugf("Global hook '%s' kube config global values stay unchanged:\n%s", utils.ValuesToString(h.moduleManager.kubeGlobalConfigValues)) + log.Debugf("Global hook '%s' kube config global values stay unchanged:\n%s", utils.ValuesToString(h.moduleManager.kubeGlobalConfigValues)) return fmt.Errorf("global hook '%s': set kube config failed: %s", h.Name, err) } h.moduleManager.kubeGlobalConfigValues = configValuesPatchResult.Values - rlog.Debugf("Global hook '%s': kube config global values updated:\n%s", h.Name, utils.ValuesToString(h.moduleManager.kubeGlobalConfigValues)) + log.Debugf("Global hook '%s': kube config global values updated:\n%s", h.Name, utils.ValuesToString(h.moduleManager.kubeGlobalConfigValues)) } } @@ -138,7 +138,7 @@ func (h *GlobalHook) run(bindingType BindingType, context []BindingContext) erro } if valuesPatchResult.ValuesChanged { h.moduleManager.globalDynamicValuesPatches = utils.AppendValuesPatch(h.moduleManager.globalDynamicValuesPatches, valuesPatchResult.ValuesPatch) - rlog.Debugf("Global hook '%s': global values updated:\n%s", h.Name, utils.ValuesToString(h.values())) + log.Debugf("Global hook '%s': global values updated:\n%s", h.Name, utils.ValuesToString(h.values())) } } @@ -216,7 +216,7 @@ func (h *GlobalHook) prepareConfigValuesYamlFile() (string, error) { return "", err } - rlog.Debugf("Prepared global hook %s config values:\n%s", h.Name, utils.ValuesToString(values)) + log.Debugf("Prepared global hook %s config values:\n%s", h.Name, utils.ValuesToString(values)) return path, nil } @@ -231,7 +231,7 @@ func (h *GlobalHook) prepareConfigValuesJsonFile() (string, error) { return "", err } - rlog.Debugf("Prepared global hook %s config values:\n%s", h.Name, utils.ValuesToString(values)) + log.Debugf("Prepared global hook %s config values:\n%s", h.Name, utils.ValuesToString(values)) return path, nil } @@ -246,7 +246,7 @@ func (h *GlobalHook) prepareValuesYamlFile() (string, error) { return "", err } - rlog.Debugf("Prepared global hook %s values:\n%s", h.Name, utils.ValuesToString(values)) + log.Debugf("Prepared global hook %s values:\n%s", h.Name, utils.ValuesToString(values)) return path, nil } @@ -261,7 +261,7 @@ func (h *GlobalHook) prepareValuesJsonFile() (string, error) { return "", err } - rlog.Debugf("Prepared global hook %s values:\n%s", h.Name, utils.ValuesToString(values)) + log.Debugf("Prepared global hook %s values:\n%s", h.Name, utils.ValuesToString(values)) return path, nil } @@ -275,7 +275,7 @@ func (h *GlobalHook) prepareBindingContextJsonFile(context interface{}) (string, return "", err } - rlog.Debugf("Prepared global hook %s binding context:\n%s", h.Name, utils_data.YamlToString(context)) + log.Debugf("Prepared global hook %s binding context:\n%s", h.Name, utils_data.YamlToString(context)) return path, nil } diff --git a/pkg/module_manager/hook.go b/pkg/module_manager/hook.go index c8d47b42..65896f49 100644 --- a/pkg/module_manager/hook.go +++ b/pkg/module_manager/hook.go @@ -7,7 +7,7 @@ import ( "sort" "github.com/kennygrant/sanitize" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" utils_file "github.com/flant/shell-operator/pkg/utils/file" ) @@ -48,7 +48,7 @@ func (h *CommonHook) GetPath() string { func SearchGlobalHooks(hooksDir string) (hooks []*GlobalHook, err error) { - rlog.Debug("INIT: search global hooks...") + log.Debug("INIT: search global hooks...") if _, err := os.Stat(hooksDir); os.IsNotExist(err) { return nil, nil @@ -63,7 +63,7 @@ func SearchGlobalHooks(hooksDir string) (hooks []*GlobalHook, err error) { // sort hooks by path sort.Strings(hooksRelativePaths) - rlog.Debugf(" Hook paths: %+v", hooksRelativePaths) + log.Debugf(" Hook paths: %+v", hooksRelativePaths) for _, hookPath := range hooksRelativePaths { hookName, err := filepath.Rel(hooksDir, hookPath) @@ -71,7 +71,7 @@ func SearchGlobalHooks(hooksDir string) (hooks []*GlobalHook, err error) { return nil, err } - rlog.Infof("INIT: global hook '%s'", hookName) + log.Infof("INIT: global hook '%s'", hookName) globalHook := NewGlobalHook(hookName, hookPath) @@ -82,7 +82,7 @@ func SearchGlobalHooks(hooksDir string) (hooks []*GlobalHook, err error) { } func SearchModuleHooks(module *Module) (hooks []*ModuleHook, err error) { - rlog.Infof("INIT: module '%s' hooks ...", module.Name) + log.Infof("INIT: module '%s' hooks ...", module.Name) hooksDir := filepath.Join(module.Path, "hooks") if _, err := os.Stat(hooksDir); os.IsNotExist(err) { @@ -98,7 +98,7 @@ func SearchModuleHooks(module *Module) (hooks []*ModuleHook, err error) { // sort hooks by path sort.Strings(hooksRelativePaths) - rlog.Debugf(" Hook paths: %+v", hooksRelativePaths) + log.Debugf(" Hook paths: %+v", hooksRelativePaths) for _, hookPath := range hooksRelativePaths { hookName, err := filepath.Rel(filepath.Dir(module.Path), hookPath) @@ -106,7 +106,7 @@ func SearchModuleHooks(module *Module) (hooks []*ModuleHook, err error) { return nil, err } - rlog.Infof("INIT: hook '%s' ...", hookName) + log.Infof("INIT: hook '%s' ...", hookName) moduleHook := NewModuleHook(hookName, hookPath) moduleHook.WithModule(module) @@ -152,7 +152,7 @@ func LoadGlobalHooksConfig(hooks []*GlobalHook) error { func (mm *MainModuleManager) RegisterGlobalHooks() error { - rlog.Debug("INIT: global hooks") + log.Debug("INIT: global hooks") mm.globalHooksOrder = make(map[BindingType][]*GlobalHook) mm.globalHooksByName = make(map[string]*GlobalHook) @@ -181,11 +181,11 @@ func (mm *MainModuleManager) RegisterGlobalHooks() error { func (mm *MainModuleManager) RegisterModuleHooks(module *Module) error { if _, ok := mm.modulesHooksOrderByName[module.Name]; ok { - rlog.Debugf("INIT: module '%s' hooks: already initialized", module.Name) + log.Debugf("INIT: module '%s' hooks: already initialized", module.Name) return nil } - rlog.Infof("INIT: module '%s' hooks ...", module.Name) + log.Infof("INIT: module '%s' hooks ...", module.Name) hooks, err := SearchModuleHooks(module) if err != nil { diff --git a/pkg/module_manager/hook/kube_event/hooks_controller.go b/pkg/module_manager/hook/kube_event/hooks_controller.go index 32631931..dc23de23 100644 --- a/pkg/module_manager/hook/kube_event/hooks_controller.go +++ b/pkg/module_manager/hook/kube_event/hooks_controller.go @@ -3,6 +3,8 @@ package kube_event import ( "fmt" + log "github.com/sirupsen/logrus" + "github.com/flant/shell-operator/pkg/hook" "github.com/flant/shell-operator/pkg/hook/kube_event" "github.com/flant/shell-operator/pkg/kube_events_manager" @@ -59,7 +61,8 @@ func (c *kubernetesHooksController) EnableGlobalHooks() error { globalHook, _ := c.moduleManager.GetGlobalHook(globalHookName) for _, config := range globalHook.Config.OnKubernetesEvents { - err := c.kubeEventsManager.AddMonitor("", config.Monitor) + logEntry := log.WithField("hook", globalHook.Name).WithField("hook.type", "global") + err := c.kubeEventsManager.AddMonitor("", config.Monitor, logEntry) if err != nil { return fmt.Errorf("run kube monitor for hook %s: %s", globalHook.Name, err) } @@ -69,16 +72,6 @@ func (c *kubernetesHooksController) EnableGlobalHooks() error { AllowFailure: config.AllowFailure, } } - // - //for _, desc := range MakeKubeEventHookDescriptors(globalHook, &globalHook.Config.HookConfig) { - // configId, err := c.kubeEventsManager.Run(desc.EventTypes, desc.Kind, desc.Namespace, desc.Selector, desc.ObjectName, desc.JqFilter, desc.Debug) - // if err != nil { - // return err - // } - // c.GlobalHooks[configId] = desc - // - // rlog.Debugf("MAIN: run informer %s for global hook %s", configId, globalHook.Name) - //} } return nil @@ -102,7 +95,8 @@ func (c *kubernetesHooksController) EnableModuleHooks(moduleName string) error { moduleHook, _ := c.moduleManager.GetModuleHook(moduleHookName) for _, config := range moduleHook.Config.OnKubernetesEvents { - err := c.kubeEventsManager.AddMonitor("", config.Monitor) + logEntry := log.WithField("hook", moduleHook.Name).WithField("hook.type", "module").WithField("module", moduleHook.Module.Name) + err := c.kubeEventsManager.AddMonitor("", config.Monitor, logEntry) if err != nil { return fmt.Errorf("run kube monitor for hook %s: %s", moduleHook.Name, err) } @@ -112,16 +106,6 @@ func (c *kubernetesHooksController) EnableModuleHooks(moduleName string) error { AllowFailure: config.AllowFailure, } } - - //for _, desc := range MakeKubeEventHookDescriptors(moduleHook, &moduleHook.Config.HookConfig) { - // configId, err := c.kubeEventsManager.Run(desc.EventTypes, desc.Kind, desc.Namespace, desc.Selector, desc.ObjectName, desc.JqFilter, desc.Debug) - // if err != nil { - // return err - // } - // c.ModuleHooks[configId] = desc - // - // rlog.Debugf("MAIN: run informer %s for module hook %s", configId, moduleHook.Name) - //} } c.EnabledModules = append(c.EnabledModules, moduleName) diff --git a/pkg/module_manager/hook/schedule/hooks_controller.go b/pkg/module_manager/hook/schedule/hooks_controller.go index ef0d2e37..652e59ae 100644 --- a/pkg/module_manager/hook/schedule/hooks_controller.go +++ b/pkg/module_manager/hook/schedule/hooks_controller.go @@ -3,9 +3,9 @@ package schedule import ( "fmt" - "github.com/flant/shell-operator/pkg/hook" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" + "github.com/flant/shell-operator/pkg/hook" "github.com/flant/shell-operator/pkg/hook/schedule" "github.com/flant/shell-operator/pkg/schedule_manager" @@ -58,10 +58,10 @@ func (c *scheduleHooksController) UpdateScheduleHooks() { for _, scheduleCfg := range globalHook.Config.Schedules { _, err := c.scheduleManager.Add(scheduleCfg.Crontab) if err != nil { - rlog.Errorf("Schedule: cannot add '%s' for hook '%s': %s", scheduleCfg.Crontab, hookName, err) + log.Errorf("Schedule: cannot add '%s' for hook '%s': %s", scheduleCfg.Crontab, hookName, err) continue } - rlog.Debugf("Schedule: add '%s' for hook '%s'", scheduleCfg.Crontab, hookName) + log.Debugf("Schedule: add '%s' for hook '%s'", scheduleCfg.Crontab, hookName) newGlobalHooks.AddHook(globalHook.Name, scheduleCfg) } } @@ -77,10 +77,10 @@ func (c *scheduleHooksController) UpdateScheduleHooks() { for _, scheduleCfg := range moduleHook.Config.Schedules { _, err := c.scheduleManager.Add(scheduleCfg.Crontab) if err != nil { - rlog.Errorf("Schedule: cannot add '%s' for hook '%s': %s", scheduleCfg.Crontab, moduleHookName, err) + log.Errorf("Schedule: cannot add '%s' for hook '%s': %s", scheduleCfg.Crontab, moduleHookName, err) continue } - rlog.Debugf("Schedule: add '%s' for hook '%s'", scheduleCfg.Crontab, moduleHookName) + log.Debugf("Schedule: add '%s' for hook '%s'", scheduleCfg.Crontab, moduleHookName) newModuleHooks.AddHook(moduleHook.Name, scheduleCfg) } } @@ -121,7 +121,7 @@ func (c *scheduleHooksController) HandleEvent(crontab string) ([]task.Task, erro for _, scheduleHook := range scheduleGlobalHooks { _, err := c.moduleManager.GetGlobalHook(scheduleHook.HookName) if err != nil { - rlog.Errorf("Possible a bug: global hook '%s' is registered for schedule but not found", scheduleHook.HookName) + log.Errorf("Possible a bug: global hook '%s' is registered for schedule but not found", scheduleHook.HookName) continue } newTask := task.NewTask(task.GlobalHookRun, scheduleHook.HookName). @@ -137,7 +137,7 @@ func (c *scheduleHooksController) HandleEvent(crontab string) ([]task.Task, erro for _, scheduleHook := range scheduleModuleHooks { _, err := c.moduleManager.GetModuleHook(scheduleHook.HookName) if err != nil { - rlog.Errorf("Possible a bug: module hook '%s' is registered for schedule but not found", scheduleHook.HookName) + log.Errorf("Possible a bug: module hook '%s' is registered for schedule but not found", scheduleHook.HookName) continue } newTask := task.NewTask(task.ModuleHookRun, scheduleHook.HookName). diff --git a/pkg/module_manager/hook_executor.go b/pkg/module_manager/hook_executor.go index cba46f06..3ab68831 100644 --- a/pkg/module_manager/hook_executor.go +++ b/pkg/module_manager/hook_executor.go @@ -5,7 +5,7 @@ import ( "os" "strings" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" "github.com/flant/shell-operator/pkg/executor" @@ -45,7 +45,7 @@ func (e *HookExecutor) Run() (patches map[utils.ValuesPatchType]*utils.ValuesPat for envName, filePath := range tmpFiles { envs = append(envs, fmt.Sprintf("%s=%s", envName, filePath)) } - envs = append(envs, helm.Client.CommandEnv()...) + envs = append(envs, helm.NewHelmCli(nil).CommandEnv()...) cmd := executor.MakeCommand("", e.Hook.GetPath(), []string{}, envs) @@ -70,20 +70,20 @@ func (e *HookExecutor) Run() (patches map[utils.ValuesPatchType]*utils.ValuesPat func (e *HookExecutor) Config() (configOutput []byte, err error) { envs := []string{} envs = append(envs, os.Environ()...) - envs = append(envs, helm.Client.CommandEnv()...) + envs = append(envs, helm.NewHelmCli(nil).CommandEnv()...) cmd := executor.MakeCommand("", e.Hook.GetPath(), []string{"--config"}, envs) - rlog.Debugf("Executing hook in %s: '%s'", cmd.Dir, strings.Join(cmd.Args, " ")) + log.Debugf("Executing hook in %s: '%s'", cmd.Dir, strings.Join(cmd.Args, " ")) cmd.Stdout = nil output, err := executor.Output(cmd) if err != nil { - rlog.Errorf("Hook '%s' config failed: %v, output:\n%s", e.Hook.GetName(), err, string(output)) + log.Errorf("Hook '%s' config failed: %v, output:\n%s", e.Hook.GetName(), err, string(output)) return nil, fmt.Errorf("%s FAILED: %s", e.Hook.GetName(), err) } - rlog.Debugf("Hook '%s' config output:\n%s", e.Hook.GetName(), string(output)) + log.Debugf("Hook '%s' config output:\n%s", e.Hook.GetName(), string(output)) return output, nil } diff --git a/pkg/module_manager/module.go b/pkg/module_manager/module.go index 9cbed3bd..1cccec46 100644 --- a/pkg/module_manager/module.go +++ b/pkg/module_manager/module.go @@ -10,7 +10,7 @@ import ( "github.com/kennygrant/sanitize" "github.com/otiai10/copy" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" "github.com/flant/shell-operator/pkg/executor" @@ -72,24 +72,27 @@ func (m *Module) Run(onStartup bool) error { return nil } +// TODO LOG: add field 'on startup' // Delete removes helm release if it exists and runs afterDeleteHelm hooks. // It is a handler for MODULE_DELETE task. func (m *Module) Delete() error { + logEntry := log.WithField("module", m.Name).WithField("phase", "delete") + // Если есть chart, но нет релиза — warning // если нет чарта — молча перейти к хукам // если есть и chart и релиз — удалить chartExists, _ := m.checkHelmChart() if chartExists { - releaseExists, err := helm.Client.IsReleaseExists(m.generateHelmReleaseName()) + releaseExists, err := helm.NewHelmCli(logEntry).IsReleaseExists(m.generateHelmReleaseName()) if !releaseExists { if err != nil { - rlog.Warnf("Module delete: Cannot find helm release '%s' for module '%s'. Helm error: %s", m.generateHelmReleaseName(), m.Name, err) + logEntry.Warnf("Cannot find helm release '%s' for module '%s'. Helm error: %s", m.generateHelmReleaseName(), m.Name, err) } else { - rlog.Warnf("Module delete: Cannot find helm release '%s' for module '%s'.", m.generateHelmReleaseName(), m.Name) + logEntry.Warnf("Cannot find helm release '%s' for module '%s'.", m.generateHelmReleaseName(), m.Name) } } else { // Chart and release are existed, so run helm delete command - err := helm.Client.DeleteRelease(m.generateHelmReleaseName()) + err := helm.NewHelmCli(logEntry).DeleteRelease(m.generateHelmReleaseName()) if err != nil { return err } @@ -104,17 +107,18 @@ func (m *Module) cleanup() error { chartExists, err := m.checkHelmChart() if !chartExists { if err != nil { - rlog.Debugf("MODULE '%s': cleanup is not needed: %s", m.Name, err) + log.Debugf("MODULE '%s': cleanup is not needed: %s", m.Name, err) return nil } } - //rlog.Infof("MODULE '%s': cleanup helm revisions...", m.Name) - if err := helm.Client.DeleteSingleFailedRevision(m.generateHelmReleaseName()); err != nil { + logEntry := log.WithField("module", m.Name).WithField("phase", "run") + + if err := helm.NewHelmCli(logEntry).DeleteSingleFailedRevision(m.generateHelmReleaseName()); err != nil { return err } - if err := helm.Client.DeleteOldFailedRevisions(m.generateHelmReleaseName()); err != nil { + if err := helm.NewHelmCli(logEntry).DeleteOldFailedRevisions(m.generateHelmReleaseName()); err != nil { return err } @@ -122,10 +126,12 @@ func (m *Module) cleanup() error { } func (m *Module) runHelmInstall() error { + logEntry := log.WithField("module", m.Name).WithField("phase", "run") + chartExists, err := m.checkHelmChart() if !chartExists { if err != nil { - rlog.Debugf("Module '%s': no Chart.yaml, helm is not needed: %s", m.Name, err) + logEntry.Debugf("no Chart.yaml, helm is not needed: %s", err) return nil } } @@ -167,20 +173,22 @@ func (m *Module) runHelmInstall() error { doRelease := true - isReleaseExists, err := helm.Client.IsReleaseExists(helmReleaseName) + helmClient := helm.NewHelmCli(logEntry) + + isReleaseExists, err := helmClient.IsReleaseExists(helmReleaseName) if err != nil { return err } if isReleaseExists { - _, status, err := helm.Client.LastReleaseStatus(helmReleaseName) + _, status, err := helmClient.LastReleaseStatus(helmReleaseName) if err != nil { return err } // Skip helm release for unchanged modules only for non FAILED releases if status != "FAILED" { - releaseValues, err := helm.Client.GetReleaseValues(helmReleaseName) + releaseValues, err := helmClient.GetReleaseValues(helmReleaseName) if err != nil { return err } @@ -189,9 +197,9 @@ func (m *Module) runHelmInstall() error { if recordedChecksumStr, ok := recordedChecksum.(string); ok { if recordedChecksumStr == checksum { doRelease = false - rlog.Infof("MODULE_RUN '%s': helm release '%s' checksum '%s' is not changed: skip helm upgrade", m.Name, helmReleaseName, checksum) + logEntry.Infof("helm release '%s' checksum '%s' is not changed: skip helm upgrade", helmReleaseName, checksum) } else { - rlog.Debugf("MODULE_RUN '%s': helm release '%s' checksum '%s' is changed to '%s': upgrade helm release", m.Name, helmReleaseName, recordedChecksumStr, checksum) + logEntry.Debugf("helm release '%s' checksum '%s' is changed to '%s': upgrade helm release", helmReleaseName, recordedChecksumStr, checksum) } } } @@ -199,9 +207,9 @@ func (m *Module) runHelmInstall() error { } if doRelease { - rlog.Debugf("MODULE_RUN '%s': helm release '%s' checksum '%s': installing/upgrading release", m.Name, helmReleaseName, checksum) + logEntry.Debugf("helm release '%s' checksum '%s': installing/upgrading release", helmReleaseName, checksum) - return helm.Client.UpgradeRelease( + return helmClient.UpgradeRelease( helmReleaseName, runChartPath, []string{valuesPath}, []string{fmt.Sprintf("_addonOperatorModuleChecksum=%s", checksum)}, @@ -209,7 +217,7 @@ func (m *Module) runHelmInstall() error { app.Namespace, ) } else { - rlog.Debugf("MODULE_RUN '%s': helm release '%s' checksum '%s': release install/upgrade is skipped", m.Name, helmReleaseName, checksum) + logEntry.Debugf("helm release '%s' checksum '%s': release install/upgrade is skipped", helmReleaseName, checksum) } return nil @@ -253,7 +261,7 @@ func (m *Module) prepareConfigValuesYamlFile() (string, error) { return "", err } - rlog.Debugf("Prepared module %s config values:\n%s", m.Name, utils.ValuesToString(values)) + log.Debugf("Prepared module %s config values:\n%s", m.Name, utils.ValuesToString(values)) return path, nil } @@ -268,7 +276,7 @@ func (m *Module) prepareConfigValuesJsonFile() (string, error) { return "", err } - rlog.Debugf("Prepared module %s config values:\n%s", m.Name, utils.ValuesToString(values)) + log.Debugf("Prepared module %s config values:\n%s", m.Name, utils.ValuesToString(values)) return path, nil } @@ -283,7 +291,7 @@ func (m *Module) prepareValuesYamlFile() (string, error) { return "", err } - rlog.Debugf("Prepared module %s values:\n%s", m.Name, utils.ValuesToString(values)) + log.Debugf("Prepared module %s values:\n%s", m.Name, utils.ValuesToString(values)) return path, nil } @@ -296,7 +304,7 @@ func (m *Module) prepareValuesJsonFileWith(values utils.Values) (string, error) return "", err } - rlog.Debugf("Prepared module %s values:\n%s", m.Name, utils.ValuesToString(values)) + log.Debugf("Prepared module %s values:\n%s", m.Name, utils.ValuesToString(values)) return path, nil } @@ -433,7 +441,7 @@ func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string) (bool, f, err := os.Stat(enabledScriptPath) if os.IsNotExist(err) { - rlog.Debugf("MODULE '%s': ENABLED. Enabled script is not exist!", m.Name) + log.Debugf("MODULE '%s': ENABLED. Enabled script is not exist!", m.Name) return true, nil } else if err != nil { return false, err @@ -458,7 +466,7 @@ func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string) (bool, return false, err } - rlog.Infof("MODULE '%s': run enabled script '%s'...", m.Name, enabledScriptPath) + log.Infof("MODULE '%s': run enabled script '%s'...", m.Name, enabledScriptPath) envs := make([]string, 0) envs = append(envs, os.Environ()...) @@ -478,18 +486,18 @@ func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string) (bool, } if moduleEnabled { - rlog.Debugf("Module '%s' ENABLED with script. Preceding: %s", m.Name, precedingEnabledModules) + log.Debugf("Module '%s' ENABLED with script. Preceding: %s", m.Name, precedingEnabledModules) return true, nil } - rlog.Debugf("Module '%s' DISABLED with script. Preceding: %s ", m.Name, precedingEnabledModules) + log.Debugf("Module '%s' DISABLED with script. Preceding: %s ", m.Name, precedingEnabledModules) return false, nil } // initModulesIndex load all available modules from modules directory // FIXME: Only 000-name modules are loaded, allow non-prefixed modules. func (mm *MainModuleManager) initModulesIndex() error { - rlog.Debug("INIT: Search modules ...") + log.Debug("INIT: Search modules ...") files, err := ioutil.ReadDir(mm.ModulesDir) // returns a list of modules sorted by filename if err != nil { @@ -510,7 +518,7 @@ func (mm *MainModuleManager) initModulesIndex() error { matchRes := validModuleName.FindStringSubmatch(file.Name()) if matchRes != nil { moduleName := matchRes[1] - rlog.Infof("INIT: Register module '%s'", moduleName) + log.Infof("INIT: Register module '%s'", moduleName) modulePath := filepath.Join(mm.ModulesDir, file.Name()) @@ -533,7 +541,7 @@ func (mm *MainModuleManager) initModulesIndex() error { } } - rlog.Debugf("INIT: initModulesIndex registered modules: %v", mm.allModulesByName) + log.Debugf("INIT: initModulesIndex registered modules: %v", mm.allModulesByName) if len(badModulesDirs) > 0 { return fmt.Errorf("found directories not matched regex '%s': %s", validModuleName, strings.Join(badModulesDirs, ", ")) @@ -549,13 +557,13 @@ func (m *Module) loadStaticValues() (err error) { if err != nil { return err } - rlog.Debugf("module %s common static values: %s", m.Name, utils.ValuesToString(m.CommonStaticConfig.Values)) + log.Debugf("module %s common static values: %s", m.Name, utils.ValuesToString(m.CommonStaticConfig.Values)) valuesYamlPath := filepath.Join(m.Path, "values.yaml") if _, err := os.Stat(valuesYamlPath); os.IsNotExist(err) { m.StaticConfig = utils.NewModuleConfig(m.Name) - rlog.Debugf("module %s is static disabled: no values.yaml exists", m.Name) + log.Debugf("module %s is static disabled: no values.yaml exists", m.Name) return nil } @@ -568,14 +576,14 @@ func (m *Module) loadStaticValues() (err error) { if err != nil { return err } - rlog.Debugf("module %s static values: %s", m.Name, utils.ValuesToString(m.StaticConfig.Values)) + log.Debugf("module %s static values: %s", m.Name, utils.ValuesToString(m.StaticConfig.Values)) return nil } func (mm *MainModuleManager) loadCommonStaticValues() (error) { valuesPath := filepath.Join(mm.ModulesDir, "values.yaml") if _, err := os.Stat(valuesPath); os.IsNotExist(err) { - rlog.Debugf("No common static values file: %s", err) + log.Debugf("No common static values file: %s", err) return nil } @@ -600,7 +608,7 @@ func (mm *MainModuleManager) loadCommonStaticValues() (error) { mm.globalCommonStaticValues = utils.GetGlobalValues(values) - rlog.Debugf("Initialized global values from common static values:\n%s", utils.ValuesToString(mm.globalCommonStaticValues)) + log.Debugf("Initialized global values from common static values:\n%s", utils.ValuesToString(mm.globalCommonStaticValues)) return nil } diff --git a/pkg/module_manager/module_hook.go b/pkg/module_manager/module_hook.go index 3882ab41..909e4b88 100644 --- a/pkg/module_manager/module_hook.go +++ b/pkg/module_manager/module_hook.go @@ -5,7 +5,7 @@ import ( "fmt" "path/filepath" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" hook2 "github.com/flant/shell-operator/pkg/hook" utils_data "github.com/flant/shell-operator/pkg/utils/data" @@ -101,7 +101,7 @@ func (h *ModuleHook) handleModuleValuesPatch(currentValues utils.Values, valuesP func (h *ModuleHook) run(bindingType BindingType, context []BindingContext) error { moduleName := h.Module.Name - rlog.Infof("Running module hook '%s' binding '%s' ...", h.Name, bindingType) + log.Infof("Running module hook '%s' binding '%s' ...", h.Name, bindingType) // Convert context for version versionedContext := make([]interface{}, 0, len(context)) @@ -129,12 +129,12 @@ func (h *ModuleHook) run(bindingType BindingType, context []BindingContext) erro if configValuesPatchResult.ValuesChanged { err := h.moduleManager.kubeConfigManager.SetKubeModuleValues(moduleName, configValuesPatchResult.Values) if err != nil { - rlog.Debugf("Module hook '%s' kube module config values stay unchanged:\n%s", utils.ValuesToString(h.moduleManager.kubeModulesConfigValues[moduleName])) + log.Debugf("Module hook '%s' kube module config values stay unchanged:\n%s", utils.ValuesToString(h.moduleManager.kubeModulesConfigValues[moduleName])) return fmt.Errorf("module hook '%s': set kube module config failed: %s", h.Name, err) } h.moduleManager.kubeModulesConfigValues[moduleName] = configValuesPatchResult.Values - rlog.Debugf("Module hook '%s': kube module '%s' config values updated:\n%s", h.Name, moduleName, utils.ValuesToString(h.moduleManager.kubeModulesConfigValues[moduleName])) + log.Debugf("Module hook '%s': kube module '%s' config values updated:\n%s", h.Name, moduleName, utils.ValuesToString(h.moduleManager.kubeModulesConfigValues[moduleName])) } } @@ -146,7 +146,7 @@ func (h *ModuleHook) run(bindingType BindingType, context []BindingContext) erro } if valuesPatchResult.ValuesChanged { h.moduleManager.modulesDynamicValuesPatches[moduleName] = utils.AppendValuesPatch(h.moduleManager.modulesDynamicValuesPatches[moduleName], valuesPatchResult.ValuesPatch) - rlog.Debugf("Module hook '%s': dynamic module '%s' values updated:\n%s", h.Name, moduleName, utils.ValuesToString(h.values())) + log.Debugf("Module hook '%s': dynamic module '%s' values updated:\n%s", h.Name, moduleName, utils.ValuesToString(h.values())) } } @@ -219,7 +219,7 @@ func (h *ModuleHook) prepareBindingContextJsonFile(context interface{}) (string, return "", err } - rlog.Debugf("Prepared module %s hook %s binding context:\n%s", h.Module.SafeName(), h.Name, utils_data.YamlToString(context)) + log.Debugf("Prepared module %s hook %s binding context:\n%s", h.Module.SafeName(), h.Name, utils_data.YamlToString(context)) return path, nil } diff --git a/pkg/module_manager/module_manager.go b/pkg/module_manager/module_manager.go index 46d2d7d1..7117e7b1 100644 --- a/pkg/module_manager/module_manager.go +++ b/pkg/module_manager/module_manager.go @@ -6,7 +6,9 @@ import ( "sort" "strings" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" + + hook_config "github.com/flant/shell-operator/pkg/hook" hook_config "github.com/flant/shell-operator/pkg/hook" @@ -190,7 +192,7 @@ type Event struct { // Init loads global hooks configs, searches for modules, loads values and calculates enabled modules func Init() { - rlog.Debug("INIT: module_manager") + log.WithField("operator.phase", "init").Debug("INIT: module_manager") EventCh = make(chan Event, 1) return @@ -226,8 +228,8 @@ func NewMainModuleManager() *MainModuleManager { // determineEnableStateWithScript runs enable script for each module that is enabled by config. // Enable script receives a list of previously enabled modules. func (mm *MainModuleManager) determineEnableStateWithScript(enabledByConfig []string) ([]string, error) { + log.Debugf("Run enable scripts for modules: %+v", enabledByConfig) enabledModules := make([]string, 0) - //rlog.Infof("Run enable scripts for modules list: %s", enabledByConfig) for _, name := range utils.SortByReference(enabledByConfig, mm.allModulesNamesInOrder) { module := mm.allModulesByName[name] @@ -241,7 +243,7 @@ func (mm *MainModuleManager) determineEnableStateWithScript(enabledByConfig []st } } - //rlog.Info("Modules enabled with script: %s", enabledModules) + log.Debugf("Modules enabled with script: %+v", enabledModules) return enabledModules, nil } @@ -254,7 +256,7 @@ type kubeUpdate struct { } func (mm *MainModuleManager) applyKubeUpdate(kubeUpdate *kubeUpdate) error { - rlog.Debugf("Apply kubeupdate %+v", kubeUpdate) + log.Debugf("Apply kubeupdate %+v", kubeUpdate) mm.kubeGlobalConfigValues = kubeUpdate.KubeGlobalConfigValues mm.kubeModulesConfigValues = kubeUpdate.KubeModulesConfigValues mm.enabledModulesByConfig = kubeUpdate.EnabledModulesByConfig @@ -267,7 +269,9 @@ func (mm *MainModuleManager) applyKubeUpdate(kubeUpdate *kubeUpdate) error { } func (mm *MainModuleManager) handleNewKubeConfig(newConfig kube_config_manager.Config) (*kubeUpdate, error) { - rlog.Debugf("MODULE_MANAGER: handle new kube config") + logEntry := log.WithField("operator.component", "ModuleManager"). + WithField("operator.action", "handleNewKubeConfig") + logEntry.Debugf("new kube config received") res := &kubeUpdate{ KubeGlobalConfigValues: newConfig.Values, @@ -278,7 +282,7 @@ func (mm *MainModuleManager) handleNewKubeConfig(newConfig kube_config_manager.C res.EnabledModulesByConfig, res.KubeModulesConfigValues, unknown = mm.calculateEnabledModulesByConfig(newConfig.ModuleConfigs) for _, moduleConfig := range unknown { - rlog.Warnf("MODULE_MANAGER: new kube config: Ignore kube config for absent module: \n%s", + logEntry.Warnf("Ignore kube config for absent module : \n%s", moduleConfig.String(), ) } @@ -287,7 +291,10 @@ func (mm *MainModuleManager) handleNewKubeConfig(newConfig kube_config_manager.C } func (mm *MainModuleManager) handleNewKubeModuleConfigs(moduleConfigs kube_config_manager.ModuleConfigs) (*kubeUpdate, error) { - rlog.Debugf("MODULE_MANAGER handle changes in module sections") + logEntry := log.WithField("operator.component", "ModuleManager"). + WithField("operator.action", "handleNewKubeModuleConfigs") + + logEntry.Debugf("handle changes in module sections") res := &kubeUpdate{ Events: make([]Event, 0), @@ -301,7 +308,7 @@ func (mm *MainModuleManager) handleNewKubeModuleConfigs(moduleConfigs kube_confi res.EnabledModulesByConfig, res.KubeModulesConfigValues, unknown = mm.calculateEnabledModulesByConfig(moduleConfigs) for _, moduleConfig := range unknown { - rlog.Warnf("HANDLE_CM_UPD ignore module section for unknown module '%s':\n%s", + logEntry.Warnf("ignore module section for unknown module '%s':\n%s", moduleConfig.ModuleName, moduleConfig.String()) } @@ -321,23 +328,23 @@ func (mm *MainModuleManager) handleNewKubeModuleConfigs(moduleConfigs kube_confi res.EnabledModulesByConfig = utils.SortByReference(res.EnabledModulesByConfig, mm.allModulesNamesInOrder) // Run enable scripts - rlog.Infof("HANDLE_CM_UPD run `enabled` for %s", res.EnabledModulesByConfig) + logEntry.Infof("run `enabled` for %s", res.EnabledModulesByConfig) enabledModules, err := mm.determineEnableStateWithScript(res.EnabledModulesByConfig) if err != nil { return nil, err } - rlog.Infof("HANDLE_CM_UPD enabled modules %s", enabledModules) + logEntry.Infof("enabled modules: %+v", enabledModules) // Configure events if !reflect.DeepEqual(mm.enabledModulesInOrder, enabledModules) { // Enabled modules set is changed — return GlobalChanged event, that will // create a Discover task, run enabled scripts again, init new module hooks, // update mm.enabledModulesInOrder - rlog.Debugf("HANDLE_CM_UPD enabledByConfig changed from %v to %v: generate GlobalChanged event", mm.enabledModulesByConfig, res.EnabledModulesByConfig) + logEntry.Debugf("enabledByConfig changed from %v to %v: generate GlobalChanged event", mm.enabledModulesByConfig, res.EnabledModulesByConfig) res.Events = append(res.Events, Event{Type: GlobalChanged}) } else { // Enabled modules set is not changed, only values in configmap are changed. - rlog.Debugf("HANDLE_CM_UPD generate ModulesChanged events...") + logEntry.Debugf("generate ModulesChanged events...") moduleChanges := make([]ModuleChange, 0) @@ -351,7 +358,7 @@ func (mm *MainModuleManager) handleNewKubeModuleConfigs(moduleConfigs kube_confi isUpdated = moduleConfig.IsUpdated // skip not updated module configs if !isUpdated { - rlog.Debugf("HANDLE_CM_UPD ignore module '%s': kube config is not updated", name) + logEntry.Debugf("ignore module '%s': kube config is not updated", name) continue } } @@ -365,8 +372,8 @@ func (mm *MainModuleManager) handleNewKubeModuleConfigs(moduleConfigs kube_confi } if len(moduleChanges) > 0 { - rlog.Infof("HANDLE_CM_UPD fire ModulesChanged event for %d modules", len(moduleChanges)) - rlog.Debugf("HANDLE_CM_UPD event changes: %v", moduleChanges) + logEntry.Infof("fire ModulesChanged event for %d modules", len(moduleChanges)) + logEntry.Debugf("event changes: %v", moduleChanges) res.Events = append(res.Events, Event{Type: ModulesChanged, ModulesChanges: moduleChanges}) } } @@ -394,7 +401,7 @@ func (mm *MainModuleManager) calculateEnabledModulesByConfig(moduleConfigs kube_ enabled = append(enabled, moduleName) values[moduleName] = kubeConfig.Values } - rlog.Debugf("Module %s: static enabled %v, kubeConfig: enabled %v, updated %v", + log.Debugf("Module %s: static enabled %v, kubeConfig: enabled %v, updated %v", module.Name, module.StaticConfig.IsEnabled, kubeConfig.IsEnabled, @@ -404,7 +411,7 @@ func (mm *MainModuleManager) calculateEnabledModulesByConfig(moduleConfigs kube_ if isEnabled { enabled = append(enabled, moduleName) } - rlog.Debugf("Module %s: static enabled %v, no kubeConfig", module.Name, module.StaticConfig.IsEnabled) + log.Debugf("Module %s: static enabled %v, no kubeConfig", module.Name, module.StaticConfig.IsEnabled) } } @@ -421,7 +428,7 @@ func (mm *MainModuleManager) calculateEnabledModulesByConfig(moduleConfigs kube_ // Init — initialize module manager func (mm *MainModuleManager) Init() error { - rlog.Debug("INIT: MODULE_MANAGER") + log.Debug("INIT: MODULE_MANAGER") if err := mm.RegisterGlobalHooks(); err != nil { return err @@ -438,7 +445,7 @@ func (mm *MainModuleManager) Init() error { mm.enabledModulesByConfig, mm.kubeModulesConfigValues, unknown = mm.calculateEnabledModulesByConfig(kubeConfig.ModuleConfigs) for _, config := range unknown { - rlog.Warnf("INIT: MODULE_MANAGER: ignore kube config for absent module: \n%s", + log.Warnf("INIT: MODULE_MANAGER: ignore kube config for absent module: \n%s", config.String(), ) } @@ -450,14 +457,16 @@ func (mm *MainModuleManager) Init() error { func (mm *MainModuleManager) Run() { go mm.kubeConfigManager.Run() + + for { select { case <-mm.globalValuesChanged: - rlog.Debugf("MODULE_MANAGER_RUN global values") + log.Debugf("MODULE_MANAGER_RUN global values") EventCh <- Event{Type: GlobalChanged} case moduleName := <-mm.moduleValuesChanged: - rlog.Debugf("MODULE_MANAGER_RUN module '%s' values changed", moduleName) + log.Debugf("MODULE_MANAGER_RUN module '%s' values changed", moduleName) // Перезапускать enabled-скрипт не нужно, т.к. // изменение values модуля не может вызвать @@ -472,12 +481,12 @@ func (mm *MainModuleManager) Run() { case newKubeConfig := <-kube_config_manager.ConfigUpdated: handleRes, err := mm.handleNewKubeConfig(newKubeConfig) if err != nil { - rlog.Errorf("MODULE_MANAGER_RUN unable to handle kube config update: %s", err) + log.Errorf("MODULE_MANAGER_RUN unable to handle kube config update: %s", err) } if handleRes != nil { err = mm.applyKubeUpdate(handleRes) if err != nil { - rlog.Errorf("MODULE_MANAGER_RUN cannot apply kube config update: %s", err) + log.Errorf("MODULE_MANAGER_RUN cannot apply kube config update: %s", err) } } @@ -492,7 +501,7 @@ func (mm *MainModuleManager) Run() { for _, newModuleConfig := range newModuleConfigs { modulesNames = append(modulesNames, fmt.Sprintf("'%s'", newModuleConfig.ModuleName)) } - rlog.Errorf("MODULE_MANAGER_RUN unable to handle modules %s kube config update: %s", strings.Join(modulesNames, ", "), err) + log.Errorf("MODULE_MANAGER_RUN unable to handle modules %s kube config update: %s", strings.Join(modulesNames, ", "), err) } if handleRes != nil { err = mm.applyKubeUpdate(handleRes) @@ -501,23 +510,22 @@ func (mm *MainModuleManager) Run() { for _, newModuleConfig := range newModuleConfigs { modulesNames = append(modulesNames, fmt.Sprintf("'%s'", newModuleConfig.ModuleName)) } - rlog.Errorf("MODULE_MANAGER_RUN cannot apply modules %s kube config update: %s", strings.Join(modulesNames, ", "), err) + log.Errorf("MODULE_MANAGER_RUN cannot apply modules %s kube config update: %s", strings.Join(modulesNames, ", "), err) } } case <-mm.retryOnAmbigous: if len(mm.moduleConfigsUpdateBeforeAmbiguos) != 0 { - rlog.Infof("MODULE_MANAGER_RUN Retry saved moduleConfigs: %v", mm.moduleConfigsUpdateBeforeAmbiguos) + log.Infof("MODULE_MANAGER_RUN Retry saved moduleConfigs: %v", mm.moduleConfigsUpdateBeforeAmbiguos) kube_config_manager.ModuleConfigsUpdated <- mm.moduleConfigsUpdateBeforeAmbiguos } else { - rlog.Debugf("MODULE_MANAGER_RUN Retry IS NOT needed") + log.Debugf("MODULE_MANAGER_RUN Retry IS NOT needed") } } } } func (mm *MainModuleManager) Retry() { - rlog.Debugf("MODULE_MANAGER Retry on ambigous") mm.retryOnAmbigous <- true } @@ -527,7 +535,9 @@ func (mm *MainModuleManager) Retry() { // // This method requires that mm.enabledModulesByConfig and mm.kubeModulesConfigValues are updated. func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err error) { - rlog.Debugf("DISCOVER state:\n"+ + logEntry := log.WithField("operator.component", "moduleManager,discoverModulesState") + + logEntry.Debugf("DISCOVER state:\n"+ " mm.enabledModulesByConfig: %v\n"+ " mm.enabledModulesInOrder: %v\n", mm.enabledModulesByConfig, @@ -540,7 +550,7 @@ func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err er NewlyEnabledModules: []string{}, } - releasedModules, err := helm.Client.ListReleasesNames(nil) + releasedModules, err := helm.NewHelmCli(logEntry).ListReleasesNames(nil) if err != nil { return nil, err } @@ -549,7 +559,7 @@ func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err er state.ReleasedUnknownModules = utils.ListSubtract(releasedModules, mm.allModulesNamesInOrder) state.ReleasedUnknownModules = utils.SortReverse(state.ReleasedUnknownModules) if len(state.ReleasedUnknownModules) > 0 { - rlog.Infof("DISCOVER found modules with releases: %s", state.ReleasedUnknownModules) + logEntry.Infof("found modules with releases: %s", state.ReleasedUnknownModules) } // ignore unknown released modules for next operations @@ -558,9 +568,9 @@ func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err er // modules finally enabled with enable script // no need to refresh mm.enabledModulesByConfig because // it is updated before in Init or in applyKubeUpdate - rlog.Infof("DISCOVER run `enabled` for %s", mm.enabledModulesByConfig) + logEntry.Infof("run `enabled` for %s", mm.enabledModulesByConfig) enabledModules, err := mm.determineEnableStateWithScript(mm.enabledModulesByConfig) - rlog.Infof("DISCOVER enabled modules %s", enabledModules) + logEntry.Infof("enabled modules %s", enabledModules) if err != nil { return nil, err } @@ -583,7 +593,7 @@ func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err er state.ModulesToDisable = utils.ListIntersection(state.ModulesToDisable, releasedModules) state.ModulesToDisable = utils.SortReverseByReference(state.ModulesToDisable, mm.allModulesNamesInOrder) - rlog.Debugf("DISCOVER state results:\n"+ + logEntry.Debugf("DISCOVER state results:\n"+ " mm.enabledModulesByConfig: %v\n"+ " EnabledModules: %v\n"+ " ReleasedUnknownModules: %v\n"+ diff --git a/pkg/module_manager/module_manager_test.go b/pkg/module_manager/module_manager_test.go index a926ca2f..e840fcd3 100644 --- a/pkg/module_manager/module_manager_test.go +++ b/pkg/module_manager/module_manager_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "github.com/romana/rlog" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "sigs.k8s.io/yaml" @@ -31,7 +31,7 @@ func initModuleManager(t *testing.T, mm *MainModuleManager, configPath string) { var err error tempDir, err := ioutil.TempDir("", "addon-operator-") - rlog.Infof("TEMP DIR %s", tempDir) + t.Logf("TEMP DIR %s", tempDir) if err != nil { t.Fatal(err) } @@ -476,8 +476,9 @@ func Test_MainModuleManager_Get_Module(t *testing.T) { //} func Test_MainModuleManager_Get_ModuleHooksInOrder(t *testing.T) { - helm.Client = &helm.MockHelmClient{} - + helm.NewHelmCli = func(logEntry *logrus.Entry) helm.HelmClient { + return &helm.MockHelmClient{} + } mm := NewMainModuleManager() initModuleManager(t, mm, "get__module_hooks_in_order") @@ -561,7 +562,10 @@ func Test_MainModuleManager_RunModule(t *testing.T) { // TODO something wrong here with patches from afterHelm and beforeHelm hooks t.SkipNow() hc := &helm.MockHelmClient{} - helm.Client = hc + + helm.NewHelmCli = func(logEntry *logrus.Entry) helm.HelmClient { + return hc + } mm := NewMainModuleManager() @@ -608,7 +612,10 @@ func Test_MainModuleManager_DeleteModule(t *testing.T) { // TODO check afterHelmDelete patch t.SkipNow() hc := &helm.MockHelmClient{} - helm.Client = hc + + helm.NewHelmCli = func(logEntry *logrus.Entry) helm.HelmClient { + return hc + } mm := NewMainModuleManager() mm.WithKubeConfigManager(MockKubeConfigManager{}) @@ -651,7 +658,9 @@ func Test_MainModuleManager_DeleteModule(t *testing.T) { func Test_MainModuleManager_RunModuleHook(t *testing.T) { // TODO hooks not found t.SkipNow() - helm.Client = &helm.MockHelmClient{} + helm.NewHelmCli = func(logEntry *logrus.Entry) helm.HelmClient { + return &helm.MockHelmClient{} + } mm := NewMainModuleManager() mm.WithKubeConfigManager(MockKubeConfigManager{}) @@ -960,7 +969,9 @@ func Test_MainModuleManager_RunModuleHook(t *testing.T) { //} func Test_MainModuleManager_Get_GlobalHooksInOrder(t *testing.T) { - helm.Client = &helm.MockHelmClient{} + helm.NewHelmCli = func(logEntry *logrus.Entry) helm.HelmClient { + return &helm.MockHelmClient{} + } mm := NewMainModuleManager() initModuleManager(t, mm, "get__global_hooks_in_order") @@ -995,7 +1006,9 @@ func Test_MainModuleManager_Get_GlobalHooksInOrder(t *testing.T) { } func Test_MainModuleManager_Run_GlobalHook(t *testing.T) { - helm.Client = &helm.MockHelmClient{} + helm.NewHelmCli = func(logEntry *logrus.Entry) helm.HelmClient { + return &helm.MockHelmClient{} + } mm := NewMainModuleManager() mm.WithKubeConfigManager(MockKubeConfigManager{}) @@ -1171,8 +1184,10 @@ func Test_MainModuleManager_DiscoverModulesState(t *testing.T) { modulesState = nil err = nil - helm.Client = &helm.MockHelmClient{ - ReleaseNames: test.helmReleases, + helm.NewHelmCli = func(logEntry *logrus.Entry) helm.HelmClient { + return &helm.MockHelmClient{ + ReleaseNames: test.helmReleases, + } } mm = NewMainModuleManager() initModuleManager(t, mm, test.configPath) diff --git a/pkg/task/tasks_queue_dumper.go b/pkg/task/tasks_queue_dumper.go index f6529b0e..60e79376 100644 --- a/pkg/task/tasks_queue_dumper.go +++ b/pkg/task/tasks_queue_dumper.go @@ -4,7 +4,7 @@ import ( "io" "os" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" ) type TasksQueueDumper struct { @@ -40,14 +40,14 @@ func (t *TasksQueueDumper) WatchQueue() { func (t *TasksQueueDumper) DumpQueue() { f, err := os.Create(t.DumpFilePath) if err != nil { - rlog.Errorf("TasksQueueDumper: Cannot open '%s': %s\n", t.DumpFilePath, err) + log.Errorf("TasksQueueDumper: Cannot open '%s': %s\n", t.DumpFilePath, err) } _, err = io.Copy(f, t.queue.DumpReader()) if err != nil { - rlog.Errorf("TasksQueueDumper: Cannot dump tasks to '%s': %s\n", t.DumpFilePath, err) + log.Errorf("TasksQueueDumper: Cannot dump tasks to '%s': %s\n", t.DumpFilePath, err) } err = f.Close() if err != nil { - rlog.Errorf("TasksQueueDumper: Cannot close '%s': %s\n", t.DumpFilePath, err) + log.Errorf("TasksQueueDumper: Cannot close '%s': %s\n", t.DumpFilePath, err) } } diff --git a/pkg/utils/values.go b/pkg/utils/values.go index 8e09ba2f..4f83f75b 100644 --- a/pkg/utils/values.go +++ b/pkg/utils/values.go @@ -8,7 +8,7 @@ import ( "sort" "strings" - "github.com/romana/rlog" + log "github.com/sirupsen/logrus" "github.com/evanphx/json-patch" "github.com/peterbourgon/mergemap" @@ -320,7 +320,7 @@ func GetGlobalValues(values Values) Values { data := map[interface{}]interface{}{GlobalValuesKey: globalValues} v, err := NewValues(data) if err != nil { - rlog.Errorf("get global Values: %s", err) + log.Errorf("get global Values: %s", err) } return v } From 979f1535b0b4774dfac71341522b9bf078722ffc Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Thu, 17 Oct 2019 17:57:18 +0300 Subject: [PATCH 3/8] go mod tidy --- go.mod | 3 +-- go.sum | 72 +++++++++++++++++----------------------------------------- 2 files changed, 22 insertions(+), 53 deletions(-) diff --git a/go.mod b/go.mod index 2b3f769d..4eb36ce1 100644 --- a/go.mod +++ b/go.mod @@ -8,19 +8,18 @@ require ( github.com/go-openapi/spec v0.19.3 github.com/kennygrant/sanitize v1.2.4 github.com/otiai10/copy v1.0.1 + github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776 // indirect github.com/peterbourgon/mergemap v0.0.0-20130613134717-e21c03b7a721 github.com/prometheus/client_golang v1.0.0 github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734 github.com/sirupsen/logrus v1.4.2 github.com/stretchr/testify v1.4.0 - golang.org/x/tools v0.0.0-20190627033414-4874f863e654 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/satori/go.uuid.v1 v1.2.0 gopkg.in/yaml.v2 v2.2.2 k8s.io/api v0.0.0-20190409092523-d687e77c8ae9 k8s.io/apimachinery v0.0.0-20190409092423-760d1845f48b k8s.io/client-go v0.0.0-20190411052641-7a6b4715b709 - k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 sigs.k8s.io/yaml v1.1.0 ) diff --git a/go.sum b/go.sum index 4463e5e3..3ecef3cb 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +bou.ke/monkey v1.0.1 h1:zEMLInw9xvNakzUUPjfS4Ds6jYPqCFx3m7bRmG5NH2U= bou.ke/monkey v1.0.1/go.mod h1:FgHuK96Rv2Nlf+0u1OOVDpCMdsWyOFmeeketDHE7LIg= cloud.google.com/go v0.0.0-20160913182117-3b1ae45394a2/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= @@ -22,79 +23,58 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/flant/go-openapi-validate v0.19.4-0.20190926112101-38fbca4ac77f h1:HTLgtnIbx2CVK74aTk1D4XbOh6tOHsPShw7UPOJTQzM= github.com/flant/go-openapi-validate v0.19.4-0.20190926112101-38fbca4ac77f/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71 h1:qjHc7vp7uBCpMbenhkgSs2uqVkZ/oLrT3uBxD+4Xnhc= github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71/go.mod h1:KQ6PubnYxbkW8BoAXfWVra5dGuMSrec/G4ztFQEC2jU= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= github.com/go-openapi/analysis v0.19.4 h1:1TjOzrWkj+9BrjnM1yPAICbaoC0FyfD49oVkTBrSSa0= github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.2 h1:rf5ArTHmIJxyV5Oiks+Su0mUens1+AjpkPoWr5xFRcI= github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= github.com/go-openapi/runtime v0.19.4 h1:csnOgcgAiuGoM/Po7PEpKDoNulCcF3FGbSnbHfxgjMI= github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= github.com/go-openapi/strfmt v0.19.2 h1:clPGfBnJohokno0e+d7hs6Yocrzjlgz6EsQSDncCRnE= github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.3 h1:PAH/2DylwWcIU1s0Y7k3yNmeAgWOcKrNE2Q7Ww/kCg4= -github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= -github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= -github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= @@ -104,9 +84,8 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= -github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -118,6 +97,7 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= @@ -128,14 +108,16 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -149,13 +131,19 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3 h1:EooPXg51Tn+xmWPXJUGCnJhJSpeuMlBmfJVcqIRmmv8= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/otiai10/copy v1.0.1 h1:gtBjD8aq4nychvRZ2CyJvFWAw0aja+VHazDdruZKGZA= github.com/otiai10/copy v1.0.1/go.mod h1:8bMCJrAqOtN/d9oyh5HR7HhLQMvcGMpGdwRDYsfOCHc= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776 h1:o59bHXu8Ejas8Kq6pjoVJQ9/neN66SM8AKh6wI42BBs= +github.com/otiai10/curr v0.0.0-20190513014714-f5a3d24e5776/go.mod h1:3HNVkVOU7vZeFXocWuvtcS0XSFLcf2XUSDHkq9t1jU4= +github.com/otiai10/mint v1.2.3 h1:PsrRBmrxR68kyNu6YlqYHbNlItc5vOkuS6LBEsNttVA= github.com/otiai10/mint v1.2.3/go.mod h1:YnfyPNhBvnY8bW4SGQHCs/aAFhkgySlMZbrF5U0bOVw= +github.com/otiai10/mint v1.2.4 h1:DxYL0itZyPaR5Z9HILdxSoHx+gNs6Yx+neOGS3IVUk0= +github.com/otiai10/mint v1.2.4/go.mod h1:d+b7n/0R3tdyUYYylALXpWQ/kTN+QobSq/4SRGBkR3M= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterbourgon/mergemap v0.0.0-20130613134717-e21c03b7a721 h1:ArxMo6jAOO2KuRsepZ0hTaH4hZCi2CCW4P9PV59HHH0= @@ -180,11 +168,8 @@ github.com/segmentio/go-camelcase v0.0.0-20160726192923-7085f1e3c734/go.mod h1:h github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -193,6 +178,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1 h1:Sq1fR+0c58RME5EoqKdjkiQAmPjmfHlZOoRI6fTUOcs= @@ -202,7 +188,6 @@ golang.org/x/crypto v0.0.0-20181025213731-e84da0312774 h1:a4tQYYYuK9QdeO/+kEvNYy golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56 h1:ZpKuNIejY8P0ExLOVyKhb0WsgG8UdvHXe6TWjY7eL6k= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -215,9 +200,6 @@ golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 h1:bfLnR+k0tq5Lqt6dflRLcZiz6 golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1CtlaD1KilQ6iudt1aI= -golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= @@ -235,7 +217,6 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313 h1:pczuHS43Cp2ktBEEmLwScxgjWsBSzdaQiKzUyf3DTTc= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4= @@ -249,18 +230,17 @@ golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190612231717-10539ce30318/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190627033414-4874f863e654 h1:a5iTclD5417yiTAwxzAQY6HG6HeJS4MqmPCoTlbd+0I= -golang.org/x/tools v0.0.0-20190627033414-4874f863e654/go.mod h1:F+l5rz3/Uc0BJWNSxc0r6FcPZ3lxfduWytgpR5peIOQ= -golang.org/x/tools/gopls v0.1.0/go.mod h1:p8Q0IUu6EEeGxqmoN/g6Et3gReLCGA7PtNRdyOxcWJE= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -268,33 +248,23 @@ gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 h1:E846t8CnR+lv5nE+Vu gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE= gopkg.in/satori/go.uuid.v1 v1.2.0 h1:AH9uksa7bGe9rluapecRKBCpZvxaBEyu0RepitcD0Hw= gopkg.in/satori/go.uuid.v1 v1.2.0/go.mod h1:kjjdhYBBaa5W5DYP+OcVG3fRM6VWu14hqDYST4Zvw+E= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= k8s.io/api v0.0.0-20190409092523-d687e77c8ae9 h1:c9UEl5z8gk1DGh/g3snETZ+a52YeR9VdbX/3BQ4PHas= k8s.io/api v0.0.0-20190409092523-d687e77c8ae9/go.mod h1:FQEUn50aaytlU65qqBn/w+5ugllHwrBzKm7DzbnXdzE= -k8s.io/api v0.0.0-20190626000116-b178a738ed00 h1:Qqj3aerxILStcStl9mGcSbVyYuLxYDr2siLyJReTyaY= -k8s.io/api v0.0.0-20190626000116-b178a738ed00/go.mod h1:O6YAz5STgv7S1/c/XtBULGhSltH7yWEHpWvnA1mmFRg= k8s.io/apimachinery v0.0.0-20190409092423-760d1845f48b h1:fVkKJL9FIpA8LSJyHVM00MP45q1WJ7+af77vcxmQP4g= k8s.io/apimachinery v0.0.0-20190409092423-760d1845f48b/go.mod h1:FW86P8YXVLsbuplGMZeb20J3jYHscrDqw4jELaFJvRU= -k8s.io/apimachinery v0.0.0-20190624085041-961b39a1baa0 h1:7oql7STcnJ85hz3BIbasXHH/+lLLKwOdsG8vjkZc8Pc= -k8s.io/apimachinery v0.0.0-20190624085041-961b39a1baa0/go.mod h1:48PVecD7ubRgJmMRGIQfsqYu6OucVH5DzFNtACHZH8k= k8s.io/client-go v0.0.0-20190411052641-7a6b4715b709 h1:5ROFGMvCLRe8mkMzReTZI/LD8A/V6h6QgNKzHW7Bbu0= k8s.io/client-go v0.0.0-20190411052641-7a6b4715b709/go.mod h1:4IOfimLkjvlSoc9wyI1VEwkNUG20XFNp7qO6XkH2gdI= -k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= -k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= k8s.io/klog v0.0.0-20190306015804-8e90cee79f82 h1:SHucoAy7lRb+w5oC/hbXyZg+zX+Wftn6hD4tGzHCVqA= k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= -k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 h1:8r+l4bNWjRlsFYlQJnKJ2p7s1YQPj4XyXiJVqDHRx7c= k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= -k8s.io/utils v0.0.0-20190607212802-c55fbcfc754a h1:2jUDc9gJja832Ftp+QbDV0tVhQHMISFn01els+2ZAcw= -k8s.io/utils v0.0.0-20190607212802-c55fbcfc754a/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 2c5ebab5304b68d543ba1c494ae6c7a3890201e0 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Wed, 23 Oct 2019 15:18:34 +0300 Subject: [PATCH 4/8] feat: json logging --- go.mod | 2 +- go.sum | 2 + pkg/addon-operator/operator.go | 445 ++++++++++-------- pkg/helm/helm.go | 4 +- pkg/helm/helm_test.go | 24 + pkg/helm/tiller.go | 34 +- pkg/module_manager/global_hook.go | 5 +- pkg/module_manager/hook.go | 100 ++-- .../hook/kube_event/hooks_controller.go | 85 ++-- .../hook/schedule/hooks_controller.go | 20 +- pkg/module_manager/hook_executor.go | 22 +- pkg/module_manager/module.go | 139 +++--- pkg/module_manager/module_hook.go | 4 +- pkg/module_manager/module_manager.go | 78 +-- pkg/module_manager/module_manager_mock.go | 36 +- pkg/module_manager/module_manager_test.go | 3 +- pkg/task/task.go | 34 +- pkg/utils/merge_labels.go | 28 ++ 18 files changed, 605 insertions(+), 460 deletions(-) create mode 100644 pkg/utils/merge_labels.go diff --git a/go.mod b/go.mod index 4eb36ce1..0e1f2269 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.12 require ( github.com/evanphx/json-patch v4.5.0+incompatible - github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71 // branch: json_logging + github.com/flant/shell-operator v1.0.0-beta.5.0.20191023115920-4f35920cc42f // branch: json_logging github.com/go-openapi/spec v0.19.3 github.com/kennygrant/sanitize v1.2.4 github.com/otiai10/copy v1.0.1 diff --git a/go.sum b/go.sum index 3ecef3cb..fca7ddd8 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/flant/go-openapi-validate v0.19.4-0.20190926112101-38fbca4ac77f h1:HT github.com/flant/go-openapi-validate v0.19.4-0.20190926112101-38fbca4ac77f/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71 h1:qjHc7vp7uBCpMbenhkgSs2uqVkZ/oLrT3uBxD+4Xnhc= github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71/go.mod h1:KQ6PubnYxbkW8BoAXfWVra5dGuMSrec/G4ztFQEC2jU= +github.com/flant/shell-operator v1.0.0-beta.5.0.20191023115920-4f35920cc42f h1:UwikOFWT3eCWIRYNugKHdZPa19mhIMEd1VWtNXoNpos= +github.com/flant/shell-operator v1.0.0-beta.5.0.20191023115920-4f35920cc42f/go.mod h1:KQ6PubnYxbkW8BoAXfWVra5dGuMSrec/G4ztFQEC2jU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= diff --git a/pkg/addon-operator/operator.go b/pkg/addon-operator/operator.go index d8fe6967..649a8628 100644 --- a/pkg/addon-operator/operator.go +++ b/pkg/addon-operator/operator.go @@ -11,8 +11,10 @@ import ( "path" "time" + "github.com/flant/addon-operator/pkg/utils" "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" + "gopkg.in/satori/go.uuid.v1" "github.com/flant/shell-operator/pkg/hook" "github.com/flant/shell-operator/pkg/kube" @@ -73,14 +75,16 @@ var ( // // Creating an empty queue with jobs. func Init() error { - log.Debug("INIT: started") + logLabels := map[string]string { + "operator.component": "Init", + } + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) var err error cwd, err := os.Getwd() if err != nil { - log.Errorf("INIT: Cannot get current working directory of process: %s", err) - return err + return fmt.Errorf("get working directory of process: %s", err) } // TODO: check if directories are existed @@ -92,15 +96,15 @@ func Init() error { if GlobalHooksDir == "" { GlobalHooksDir = path.Join(cwd, app.GlobalHooksDir) } - log.Infof("INIT: Modules: '%s', Global hooks: '%s'", ModulesDir, GlobalHooksDir) + logEntry.Infof("Global hooks directory: %s", GlobalHooksDir) + logEntry.Infof("Modules directory: %s", ModulesDir) TempDir := app.TmpDir err = os.MkdirAll(TempDir, os.FileMode(0777)) if err != nil { - log.Errorf("INIT: Cannot create temporary dir '%s': %s", TempDir, err) - return err + return fmt.Errorf("create temp directory '%s': %s", TempDir, err) } - log.Infof("INIT: Temporary dir: %s", TempDir) + logEntry.Infof("Temp directory: %s", TempDir) // init and start metrics gathering loop @@ -111,7 +115,7 @@ func Init() error { // Initializing the empty task queue and queue dumper TasksQueue = task.NewTasksQueue() // Initializing the queue dumper, which writes queue changes to the dump file. - log.Debugf("INIT: Tasks queue dump file: '%s'", app.TasksQueueDumpFilePath) + logEntry.Infof("Queue dump file path: %s", app.TasksQueueDumpFilePath) queueWatcher := task.NewTasksQueueDumper(app.TasksQueueDumpFilePath, TasksQueue) TasksQueue.AddWatcher(queueWatcher) @@ -119,14 +123,13 @@ func Init() error { // Initializing the connection to the k8s. err = kube.Init(kube.InitOptions{}) if err != nil { - log.Errorf("INIT: Cannot initialize Kubernetes client: %s", err) - return err + return fmt.Errorf("init Kubernetes client: %s", err) } // A useful callback when addon-operator is used as library if BeforeHelmInitCb != nil { - log.Debugf("INIT: run BeforeHelmInitCallback") + logEntry.Debugf("run BeforeHelmInitCallback") BeforeHelmInitCb() } @@ -139,15 +142,13 @@ func Init() error { ProbeListenPort: app.TillerProbeListenPort, }) if err != nil { - log.Errorf("INIT: Tiller is failed to start: %s", err) - return err + return fmt.Errorf("init tiller: %s", err) } // Initializing helm client err = helm.InitClient() if err != nil { - log.Errorf("INIT: helm client: %s", err) - return err + return fmt.Errorf("init helm client: %s", err) } @@ -159,7 +160,7 @@ func Init() error { err = KubeConfigManager.Init() if err != nil { - return err + return fmt.Errorf("init kube config manager: %s", err) } module_manager.Init() @@ -168,16 +169,14 @@ func Init() error { ModuleManager.WithKubeConfigManager(KubeConfigManager) err = ModuleManager.Init() if err != nil { - log.Errorf("INIT: Cannot initialize module manager: %s", err) - return err + return fmt.Errorf("init module manager: %s", err) } // Initializing the hooks schedule. ScheduleManager, err = schedule_manager.Init() if err != nil { - log.Errorf("INIT: Cannot initialize schedule manager: %s", err) - return err + return fmt.Errorf("init schedule manager: %s", err) } ScheduleHooksController = schedule_hook.NewScheduleHooksController() @@ -191,15 +190,6 @@ func Init() error { KubernetesHooksController.WithModuleManager(ModuleManager) KubernetesHooksController.WithKubeEventsManager(KubeEventsManager) - - //// Initialize kube events - //KubeEventsManager, err = kube_events_manager.Init() - //if err != nil { - // log.Errorf("INIT: Cannot initialize kube events manager: %s", err) - // return err - //} - //KubernetesHooksController = kube_event_hook.NewKubernetesHooksController() - return nil } @@ -209,22 +199,21 @@ func Init() error { func Run() { // Loading the onStartup hooks into the queue and running all modules. // Turning tracking changes on only after startup ends. - log.Info("MAIN: Start. Trigger onStartup event.") + onStartupLabels := map[string]string {} + onStartupLabels["event.id"] = "OperatorOnStartup" TasksQueue.ChangesDisable() + CreateOnStartupTasks(onStartupLabels) + CreateReloadAllTasks(true, onStartupLabels) + TasksQueue.ChangesEnable(true) - CreateOnStartupTasks() - CreateReloadAllTasks(true) - - err := KubernetesHooksController.EnableGlobalHooks() + // Handle kubernetes events + err := KubernetesHooksController.EnableGlobalHooks(onStartupLabels) if err != nil { // Something wrong with global hook configs, cannot start informers. - log.Errorf("Start informers for global kubernetes hooks: %v", err) + log.Errorf("Create informers for global kubernetes hooks: %v", err) return } - // Start all created informers - KubeEventsManager.Start() - TasksQueue.ChangesEnable(true) go ModuleManager.Run() go ScheduleManager.Run() @@ -235,6 +224,7 @@ func Run() { // TasksRunner runs tasks from the queue. go TasksRunner() + // Health metrics RunAddonOperatorMetrics() } @@ -243,6 +233,10 @@ func ManagersEventsHandler() { select { // Event from module manager (module restart or full restart). case moduleEvent := <-module_manager.EventCh: + logLabels := map[string]string{ + "event.id": uuid.NewV4().String(), + } + eventLogEntry := log.WithField("operator.component", "handleManagerEvents") // Event from module manager can come if modules list have changed, // so event hooks need to be re-register with: // RegisterScheduledHooks() @@ -250,154 +244,102 @@ func ManagersEventsHandler() { switch moduleEvent.Type { // Some modules have changed. case module_manager.ModulesChanged: + logLabels["event.type"] = "ModulesChanged" + logEntry := eventLogEntry.WithFields(utils.LabelsToLogFields(logLabels)) for _, moduleChange := range moduleEvent.ModulesChanges { - log.Infof("EVENT ModulesChanged, type=Changed") - newTask := task.NewTask(task.ModuleRun, moduleChange.Name) + logEntry.WithField("module", moduleChange.Name).Infof("module values are changed, queue ModuleRun task", moduleChange.Name) + newTask := task.NewTask(task.ModuleRun, moduleChange.Name). + WithLogLabels(logLabels) TasksQueue.Add(newTask) - log.Infof("QUEUE add ModuleRun %s", newTask.Name) } // As module list may have changed, hook schedule index must be re-created. ScheduleHooksController.UpdateScheduleHooks() case module_manager.GlobalChanged: + logLabels["event.type"] = "GlobalChanged" + logEntry := eventLogEntry.WithFields(utils.LabelsToLogFields(logLabels)) // Global values are changed, all modules must be restarted. - log.Infof("EVENT GlobalChanged") + logEntry.Infof("global values are changed, queue ReloadAll tasks") TasksQueue.ChangesDisable() - CreateReloadAllTasks(false) + CreateReloadAllTasks(false, logLabels) TasksQueue.ChangesEnable(true) // As module list may have changed, hook schedule index must be re-created. ScheduleHooksController.UpdateScheduleHooks() case module_manager.AmbigousState: - log.Infof("EVENT AmbiguousState") - TasksQueue.ChangesDisable() // It is the error in the module manager. The task must be added to // the beginning of the queue so the module manager can restore its // state before running other queue tasks - newTask := task.NewTask(task.ModuleManagerRetry, "") + logLabels["event.type"] = "AmbigousState" + logEntry := eventLogEntry.WithFields(utils.LabelsToLogFields(logLabels)) + logEntry.Infof("module manager is in ambiguous state, queue ModuleManagerRetry task with delay") + TasksQueue.ChangesDisable() + newTask := task.NewTask(task.ModuleManagerRetry, ""). + WithLogLabels(logLabels) TasksQueue.Push(newTask) // It is the delay before retry. - TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) + TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay).WithLogLabels(logLabels)) TasksQueue.ChangesEnable(true) - log.Infof("QUEUE push ModuleManagerRetry, push FailedModuleDelay") } + case crontab := <-schedule_manager.ScheduleCh: - log.Infof("EVENT Schedule event '%s'", crontab) + logLabels := map[string]string{ + "event.id": uuid.NewV4().String(), + "binding": module_manager.ContextBindingType[module_manager.Schedule], + } + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) + logEntry.Infof("Event from '%s'", crontab) - tasks, err := ScheduleHooksController.HandleEvent(crontab) + tasks, err := ScheduleHooksController.HandleEvent(crontab, logLabels) if err != nil { - log.Errorf("MAIN_LOOP Schedule event '%s': %s", crontab, err) + logEntry.Errorf("handle schedule event '%s': %s", crontab, err) break } - for _, resTask := range tasks { - TasksQueue.Add(resTask) - log.Infof("QUEUE add %s@%s %s", resTask.GetType(), resTask.GetBinding(), resTask.GetName()) + for _, t := range tasks { + logEntry. + WithFields(utils.LabelsToLogFields(t.GetLogLabels())). + Infof("queue %s task", t.GetType()) + TasksQueue.Add(t) } case kubeEvent := <-kube_events_manager.KubeEventCh: - log.Infof("EVENT Kube event '%s'", kubeEvent.ConfigId) + logLabels := map[string]string{ + "event.id": uuid.NewV4().String(), + "binding": module_manager.ContextBindingType[module_manager.KubeEvents], + } + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) + if kubeEvent.Type == "Event" { + logEntry.Infof("%s event for %s/%s/%s", kubeEvent.Event, kubeEvent.Namespace, kubeEvent.Kind, kubeEvent.Name) + } else { + logEntry.Infof("Synchronization") + } - tasks, err := KubernetesHooksController.HandleEvent(kubeEvent) + tasks, err := KubernetesHooksController.HandleEvent(kubeEvent, logLabels) if err != nil { - log.Errorf("MAIN_LOOP error handling kube event '%s': %s", kubeEvent.ConfigId, err) + logEntry.Errorf("handle kubernetes event '%s': %s", kubeEvent.ConfigId, err) break } for _, t := range tasks { + logEntry. + WithFields(utils.LabelsToLogFields(t.GetLogLabels())). + Infof("queue %s task", t.GetType()) TasksQueue.Add(t) - log.Infof("QUEUE add %s@%s %s", t.GetType(), t.GetBinding(), t.GetName()) } + case <-ManagersEventsHandlerStopCh: - log.Infof("EVENT Stop") + log.Infof("Stop from ManagersEventsHandlerStopCh") return } } } -func runDiscoverModulesState(discoverTask task.Task) error { - modulesState, err := ModuleManager.DiscoverModulesState() - if err != nil { - return err - } - - for _, moduleName := range modulesState.EnabledModules { - isNewlyEnabled := false - for _, name := range modulesState.NewlyEnabledModules { - if name == moduleName { - isNewlyEnabled = true - break - } - } - runOnStartupHooks := discoverTask.GetOnStartupHooks() || isNewlyEnabled - - newTask := task.NewTask(task.ModuleRun, moduleName). - WithOnStartupHooks(runOnStartupHooks) - - TasksQueue.Add(newTask) - log.Infof("QUEUE add ModuleRun %s", moduleName) - } - - for _, moduleName := range modulesState.ModulesToDisable { - // TODO may be only afterHelmDelete hooks should be initialized? - // Enable module hooks on startup to run afterHelmDelete hooks - if discoverTask.GetOnStartupHooks() { - // error can be ignored, DiscoverModulesState should return existed modules - disabledModule, _ := ModuleManager.GetModule(moduleName) - if err = ModuleManager.RegisterModuleHooks(disabledModule); err != nil { - return err - } - } - newTask := task.NewTask(task.ModuleDelete, moduleName) - TasksQueue.Add(newTask) - log.Infof("QUEUE add ModuleDelete %s", moduleName) - } - - for _, moduleName := range modulesState.ReleasedUnknownModules { - newTask := task.NewTask(task.ModulePurge, moduleName) - TasksQueue.Add(newTask) - log.Infof("QUEUE add ModulePurge %s", moduleName) - } - - // Queue afterAll global hooks - afterAllHooks := ModuleManager.GetGlobalHooksInOrder(module_manager.AfterAll) - for _, hookName := range afterAllHooks { - newTask := task.NewTask(task.GlobalHookRun, hookName). - WithBinding(module_manager.AfterAll). - AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: module_manager.ContextBindingType[module_manager.AfterAll]}}) - TasksQueue.Add(newTask) - log.Debugf("QUEUE add GlobalHookRun@AfterAll '%s'", hookName) - } - if len(afterAllHooks) > 0 { - log.Infof("QUEUE add all GlobalHookRun@AfterAll") - } - - ScheduleHooksController.UpdateScheduleHooks() - - // Enable kube events hooks for newly enabled modules - // FIXME convert to a task that run after AfterHelm if there is a flag in binding config to start informers after CRD installation. - for _, moduleName := range modulesState.EnabledModules { - err = KubernetesHooksController.EnableModuleHooks(moduleName) - if err != nil { - return err - } - } - - // TODO is queue should be cleaned from hook run tasks of deleted module? - // Disable kube events hooks for newly disabled modules - for _, moduleName := range modulesState.ModulesToDisable { - err = KubernetesHooksController.DisableModuleHooks(moduleName) - if err != nil { - return err - } - } - - return nil -} // TasksRunner handle tasks in queue. // // Task handler may delay task processing by pushing delay to the queue. -// FIXME: For now, only one TaskRunner for a TasksQueue. There should be a lock between Peek and Pop to prevent Poping tasks from other TaskRunner +// FIXME: For now, only one TaskRunner for a TasksQueue. There should be a lock between Peek and Pop to prevent Poping tasks by other TaskRunner for multiple queues. func TasksRunner() { + logEntry := log.WithField("operator.component", "TaskRunner") for { if TasksQueue.IsEmpty() { time.Sleep(QueueIsEmptyDelay) @@ -408,48 +350,49 @@ func TasksRunner() { break } + taskLogEntry := logEntry.WithFields(utils.LabelsToLogFields(t.GetLogLabels())). + WithField("task.type", t.GetType()) + switch t.GetType() { case task.DiscoverModulesState: - log.Infof("TASK_RUN DiscoverModulesState") - err := runDiscoverModulesState(t) + taskLogEntry.Info("Run DiscoverModules") + err := RunDiscoverModulesState(t, t.GetLogLabels()) if err != nil { MetricsStorage.SendCounterMetric(PrefixMetric("modules_discover_errors"), 1.0, map[string]string{}) t.IncrementFailureCount() - log.Errorf("TASK_RUN %s failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetFailureCount(), err) + taskLogEntry.Errorf("DiscoverModulesState failed, queue Delay task to retry. Failed count is %d. Error: %s", t.GetFailureCount(), err) TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) - log.Infof("QUEUE push FailedModuleDelay") - break + } else { + taskLogEntry.Infof("DiscoverModulesState success") + TasksQueue.Pop() } - - TasksQueue.Pop() - case task.ModuleRun: - log.Infof("TASK_RUN ModuleRun %s", t.GetName()) - err := ModuleManager.RunModule(t.GetName(), t.GetOnStartupHooks()) + taskLogEntry.Info("Run module") + err := ModuleManager.RunModule(t.GetName(), t.GetOnStartupHooks(), t.GetLogLabels()) if err != nil { MetricsStorage.SendCounterMetric(PrefixMetric("module_run_errors"), 1.0, map[string]string{"module": t.GetName()}) t.IncrementFailureCount() - log.Errorf("TASK_RUN ModuleRun '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetName(), t.GetFailureCount(), err) + taskLogEntry.Errorf("ModuleRun failed, queue Delay task to retry. Failed count is %d. Error: %s", t.GetFailureCount(), err) TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) - log.Infof("QUEUE push FailedModuleDelay") } else { + taskLogEntry.Infof("ModuleRun success") TasksQueue.Pop() } case task.ModuleDelete: - log.Infof("TASK_RUN ModuleDelete %s", t.GetName()) - err := ModuleManager.DeleteModule(t.GetName()) + taskLogEntry.Info("Delete module") + err := ModuleManager.DeleteModule(t.GetName(), t.GetLogLabels()) if err != nil { MetricsStorage.SendCounterMetric(PrefixMetric("module_delete_errors"), 1.0, map[string]string{"module": t.GetName()}) t.IncrementFailureCount() - log.Errorf("%s '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetName(), t.GetFailureCount(), err) + taskLogEntry.Errorf("ModuleDelete failed, queue Delay task to retry. Failed count is %d. Error: %s", t.GetFailureCount(), err) TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) - log.Infof("QUEUE push FailedModuleDelay") } else { + taskLogEntry.Infof("ModuleDelete success") TasksQueue.Pop() } case task.ModuleHookRun: - log.Infof("TASK_RUN ModuleHookRun@%s %s", t.GetBinding(), t.GetName()) - err := ModuleManager.RunModuleHook(t.GetName(), t.GetBinding(), t.GetBindingContext()) + taskLogEntry.Info("Run module hooks") + err := ModuleManager.RunModuleHook(t.GetName(), t.GetBinding(), t.GetBindingContext(), t.GetLogLabels()) if err != nil { moduleHook, _ := ModuleManager.GetModuleHook(t.GetName()) hookLabel := path.Base(moduleHook.Path) @@ -457,116 +400,231 @@ func TasksRunner() { if t.GetAllowFailure() { MetricsStorage.SendCounterMetric(PrefixMetric("module_hook_allowed_errors"), 1.0, map[string]string{"module": moduleLabel, "hook": hookLabel}) + taskLogEntry.Infof("ModuleHookRun failed, but allowed to fail. Error: %v", err) TasksQueue.Pop() } else { MetricsStorage.SendCounterMetric(PrefixMetric("module_hook_errors"), 1.0, map[string]string{"module": moduleLabel, "hook": hookLabel}) t.IncrementFailureCount() - log.Errorf("%s '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetName(), t.GetFailureCount(), err) + taskLogEntry.Errorf("ModuleHookRun failed, queue Delay task to retry. Failed count is %d. Error: %s", t.GetFailureCount(), err) TasksQueue.Push(task.NewTaskDelay(FailedModuleDelay)) - log.Infof("QUEUE push FailedModuleDelay") } } else { + taskLogEntry.Infof("ModuleHookRun success") TasksQueue.Pop() } case task.GlobalHookRun: - log.Infof("TASK_RUN GlobalHookRun@%s %s", t.GetBinding(), t.GetName()) - err := ModuleManager.RunGlobalHook(t.GetName(), t.GetBinding(), t.GetBindingContext()) + taskLogEntry.Infof("Run global hook") + err := ModuleManager.RunGlobalHook(t.GetName(), t.GetBinding(), t.GetBindingContext(), t.GetLogLabels()) if err != nil { globalHook, _ := ModuleManager.GetGlobalHook(t.GetName()) hookLabel := path.Base(globalHook.Path) if t.GetAllowFailure() { MetricsStorage.SendCounterMetric(PrefixMetric("global_hook_allowed_errors"), 1.0, map[string]string{"hook": hookLabel}) + taskLogEntry.Infof("GlobalHookRun failed, but allowed to fail. Error: %v", err) TasksQueue.Pop() } else { MetricsStorage.SendCounterMetric(PrefixMetric("global_hook_errors"), 1.0, map[string]string{"hook": hookLabel}) t.IncrementFailureCount() - log.Errorf("TASK_RUN %s '%s' on '%s' failed. Will retry after delay. Failed count is %d. Error: %s", t.GetType(), t.GetName(), t.GetBinding(), t.GetFailureCount(), err) + taskLogEntry.Errorf("GlobalHookRun failed, queue Delay task to retry. Failed count is %d. Error: %s", t.GetFailureCount(), err) TasksQueue.Push(task.NewTaskDelay(FailedHookDelay)) } } else { + taskLogEntry.Infof("GlobalHookRun success") TasksQueue.Pop() } case task.ModulePurge: - log. - WithField("operator.component", "taskRunner"). - WithField("task", "ModulePurge"). - WithField("module", t.GetName()). - Debugf("run task") - logEntry := log.WithField("module", t.GetName()).WithField("phase", "purge") - // Module for purge is unknown so log deletion error is enough. - err := helm.NewHelmCli(logEntry).DeleteRelease(t.GetName()) + // Purge is for unknown modules, so error is just ignored. + taskLogEntry.Infof("Run module purge") + err := helm.NewHelmCli(taskLogEntry).DeleteRelease(t.GetName()) if err != nil { - log.Errorf("TASK_RUN %s Helm delete '%s' failed. Error: %s", t.GetType(), t.GetName(), err) + taskLogEntry.Errorf("ModulePurge failed, no retry. Error: %s", err) + } else { + taskLogEntry.Infof("ModulePurge success") } TasksQueue.Pop() case task.ModuleManagerRetry: - log. - WithField("operator.component", "taskRunner"). - WithField("task", "ModuleManagerRetry"). - WithField("module", t.GetName()). - Infof("Retry") MetricsStorage.SendCounterMetric(PrefixMetric("modules_discover_errors"), 1.0, map[string]string{}) ModuleManager.Retry() TasksQueue.Pop() - // Adding a delay before retrying module/hook task. - newTask := task.NewTaskDelay(FailedModuleDelay) - newTask.Name = t.GetName() + newTask := task.NewTaskDelay(FailedModuleDelay).WithLogLabels(t.GetLogLabels()) + taskLogEntry.Infof("Queue Delay task immediately to wait for success module discovery") TasksQueue.Push(newTask) - log.Infof("QUEUE push FailedModuleDelay") case task.Delay: - log.Infof("TASK_RUN Delay for %s", t.GetDelay().String()) + taskLogEntry.Debugf("Sleep for %s", t.GetDelay().String()) TasksQueue.Pop() time.Sleep(t.GetDelay()) case task.Stop: - log.Infof("TASK_RUN Stop: Exiting TASK_RUN loop.") + logEntry.Infof("Stop") TasksQueue.Pop() return } // Breaking, if the task queue is empty to prevent the infinite loop. if TasksQueue.IsEmpty() { - log.Debug("Task queue is empty. Will sleep now.") + logEntry.Debugf("Task queue is empty.") break } } } } -func CreateOnStartupTasks() { - log.Infof("QUEUE add all GlobalHookRun@OnStartup") +func CreateOnStartupTasks(logLabels map[string]string) { + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) onStartupHooks := ModuleManager.GetGlobalHooksInOrder(module_manager.OnStartup) for _, hookName := range onStartupHooks { + hookLogLabels := utils.MergeLabels(logLabels) + hookLogLabels["hook"] = hookName + hookLogLabels["hook.type"] = "global" + hookLogLabels["binding"] = module_manager.ContextBindingType[module_manager.OnStartup] + + logEntry.WithFields(utils.LabelsToLogFields(hookLogLabels)). + Infof("queue GlobalHookRun task") newTask := task.NewTask(task.GlobalHookRun, hookName). WithBinding(module_manager.OnStartup). + WithLogLabels(hookLogLabels). AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: module_manager.ContextBindingType[module_manager.OnStartup]}}) TasksQueue.Add(newTask) - log.Debugf("QUEUE add GlobalHookRun@OnStartup '%s'", hookName) } return } -func CreateReloadAllTasks(onStartup bool) { - log.Infof("QUEUE add all GlobalHookRun@BeforeAll, add DiscoverModulesState") +func CreateReloadAllTasks(onStartup bool, logLabels map[string]string) { + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) // Queue beforeAll global hooks. beforeAllHooks := ModuleManager.GetGlobalHooksInOrder(module_manager.BeforeAll) for _, hookName := range beforeAllHooks { + hookLogLabels := utils.MergeLabels(logLabels) + hookLogLabels["hook"] = hookName + hookLogLabels["hook.type"] = "global" + hookLogLabels["binding"] = module_manager.ContextBindingType[module_manager.BeforeAll] + + logEntry.WithFields(utils.LabelsToLogFields(hookLogLabels)). + Infof("queue GlobalHookRun task") newTask := task.NewTask(task.GlobalHookRun, hookName). WithBinding(module_manager.BeforeAll). + WithLogLabels(hookLogLabels). AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: module_manager.ContextBindingType[module_manager.BeforeAll]}}) + TasksQueue.Add(newTask) + } + + logEntry.Infof("queue DiscoverModulesState task") + discoverTask := task.NewTask(task.DiscoverModulesState, ""). + WithOnStartupHooks(onStartup). + WithLogLabels(logLabels) + TasksQueue.Add(discoverTask) +} +func RunDiscoverModulesState(discoverTask task.Task, logLabels map[string]string) error { + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) + modulesState, err := ModuleManager.DiscoverModulesState(logLabels) + if err != nil { + return err + } + + // queue ModuleRun tasks for enabled modules + for _, moduleName := range modulesState.EnabledModules { + moduleLogEntry := logEntry.WithField("module", moduleName) + moduleLogLabels := utils.MergeLabels(logLabels) + moduleLogLabels["module"] = moduleName + + // Run OnStartup hooks on application startup or if module become enabled + runOnStartupHooks := discoverTask.GetOnStartupHooks() + if !runOnStartupHooks { + for _, name := range modulesState.NewlyEnabledModules { + if name == moduleName { + runOnStartupHooks = true + break + } + } + } + + newTask := task.NewTask(task.ModuleRun, moduleName). + WithLogLabels(moduleLogLabels). + WithOnStartupHooks(runOnStartupHooks) + + moduleLogEntry.Infof("queue ModuleRun task for %s", moduleName) TasksQueue.Add(newTask) - log.Debugf("QUEUE GlobalHookRun@BeforeAll '%s'", module_manager.BeforeAll, hookName) } - TasksQueue.Add(task.NewTask(task.DiscoverModulesState, "").WithOnStartupHooks(onStartup)) + // queue ModuleDelete tasks for disabled modules + for _, moduleName := range modulesState.ModulesToDisable { + moduleLogEntry := logEntry.WithField("module", moduleName) + modLogLabels := utils.MergeLabels(logLabels) + modLogLabels["module"] = moduleName + // TODO may be only afterHelmDelete hooks should be initialized? + // Enable module hooks on startup to run afterHelmDelete hooks + if discoverTask.GetOnStartupHooks() { + // error can be ignored, DiscoverModulesState should return existed modules + disabledModule, _ := ModuleManager.GetModule(moduleName) + if err = ModuleManager.RegisterModuleHooks(disabledModule, modLogLabels); err != nil { + return err + } + } + moduleLogEntry.Infof("queue ModuleDelete task for %s", moduleName) + newTask := task.NewTask(task.ModuleDelete, moduleName). + WithLogLabels(modLogLabels) + TasksQueue.Add(newTask) + } + + // queue ModulePurge tasks for unknown modules + for _, moduleName := range modulesState.ReleasedUnknownModules { + moduleLogEntry := logEntry.WithField("module", moduleName) + newTask := task.NewTask(task.ModulePurge, moduleName). + WithLogLabels(logLabels) + TasksQueue.Add(newTask) + moduleLogEntry.Infof("queue ModulePurge task") + } + + // Queue afterAll global hooks + afterAllHooks := ModuleManager.GetGlobalHooksInOrder(module_manager.AfterAll) + for _, hookName := range afterAllHooks { + hookLogLabels := utils.MergeLabels(logLabels) + hookLogLabels["hook"] = hookName + hookLogLabels["hook.type"] = "global" + + logEntry.WithFields(utils.LabelsToLogFields(hookLogLabels)). + Infof("queue GlobalHookRun task") + newTask := task.NewTask(task.GlobalHookRun, hookName). + WithBinding(module_manager.AfterAll). + WithLogLabels(hookLogLabels). + AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: module_manager.ContextBindingType[module_manager.AfterAll]}}) + TasksQueue.Add(newTask) + } + + ScheduleHooksController.UpdateScheduleHooks() + + // Enable kube events hooks for newly enabled modules + // FIXME convert to a task that run after AfterHelm if there is a flag in binding config to start informers after CRD installation. + for _, moduleName := range modulesState.EnabledModules { + modLogLabels := utils.MergeLabels(logLabels) + modLogLabels["module"] = moduleName + err = KubernetesHooksController.EnableModuleHooks(moduleName, modLogLabels) + if err != nil { + return err + } + } + + // TODO is queue should be cleaned from hook run tasks of deleted module? + // Disable kube events hooks for newly disabled modules + for _, moduleName := range modulesState.ModulesToDisable { + modLogLabels := utils.MergeLabels(logLabels) + modLogLabels["module"] = moduleName + err = KubernetesHooksController.DisableModuleHooks(moduleName, modLogLabels) + if err != nil { + return err + } + } + + return nil } + func RunAddonOperatorMetrics() { // Addon-operator live ticks. go func() { @@ -592,6 +650,11 @@ func InitHttpServer(listenAddr string, listenPort string) error {

Addon-operator

go tool pprof goprofex http://ADDON_OPERATOR_IP:9115/debug/pprof/profile
+

+ prometheus metrics + health url + queue stats +

`)) }) @@ -606,7 +669,7 @@ func InitHttpServer(listenAddr string, listenPort string) error { }) address := fmt.Sprintf("%s:%s", listenAddr, listenPort) - log.Infof("HTTP SERVER Listening on %s", address) + log.Infof("Listen on %s", address) // Check if port is available ln, err := net.Listen("tcp", address) diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 6b353901..66a4f77a 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -48,8 +48,6 @@ var _ HelmClient = &CliHelm{} // InitClient initialize helm client func InitClient() error { - log.Info("Helm: run helm init") - cliHelm := &CliHelm{} stdout, stderr, err := cliHelm.Cmd("init", "--client-only") @@ -63,7 +61,7 @@ func InitClient() error { } stdout = strings.Join([]string{stdout, stderr}, "\n") stdout = strings.ReplaceAll(stdout, "\n", " ") - log.Infof("Helm: successfully initialized. Version: %s", stdout) + log.Infof("Helm successfully initialized. Version: %s", stdout) return nil } diff --git a/pkg/helm/helm_test.go b/pkg/helm/helm_test.go index 917b5005..638f6280 100644 --- a/pkg/helm/helm_test.go +++ b/pkg/helm/helm_test.go @@ -8,6 +8,7 @@ import ( "sort" "testing" + "github.com/flant/addon-operator/pkg/utils" log "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" @@ -17,6 +18,8 @@ import ( ) func Test_Logging(t *testing.T) { + log.SetFormatter(&log.JSONFormatter{DisableTimestamp: true}) + log.Info("Start test") logEntry1 := log.WithField("test", "helm") @@ -33,6 +36,27 @@ func Test_Logging(t *testing.T) { logEntry11.WithField("test","helm11").Infof("helmm info") + fields1 := map[string]string { + "module": "mod1", + "hook": "hook2", + "component": "main", + } + logEntry1F := logEntry1.WithFields(utils.LabelsToLogFields(fields1)) + logEntry1F.Infof("top record") + + fields2 := map[string]string { + "module":"mod2", + "event.id": "123", + } + + logEntry2F := logEntry1F.WithFields(utils.LabelsToLogFields(fields2)) + + logEntry2F.Infof("nested record") + logEntry1F.Infof("new top record") + logEntry2F.Infof("new nested record") + + + logEntry2F.WithField("result", "qwe\nfoo\bqwe").Infof("record with multiline field") } diff --git a/pkg/helm/tiller.go b/pkg/helm/tiller.go index 6b609423..00a311ec 100644 --- a/pkg/helm/tiller.go +++ b/pkg/helm/tiller.go @@ -12,6 +12,7 @@ import ( ) const TillerPath = "tiller" +const MaxHelmVersionWaits = 32 // TillerOptions type TillerOptions struct { @@ -25,6 +26,8 @@ type TillerOptions struct { // InitTillerProcess starts tiller as a subprocess. If tiller is exited, addon-operator also exits. func InitTillerProcess(options TillerOptions) error { + logEntry := log.WithField("operator.component", "tiller") + env := []string{ fmt.Sprintf("TILLER_NAMESPACE=%s", options.Namespace), fmt.Sprintf("TILLER_HISTORY_MAX=%d", options.HistoryMax), @@ -43,30 +46,40 @@ func InitTillerProcess(options TillerOptions) error { err := tillerCmd.Start() if err != nil { - log.Errorf("Tiller process not started: %v", err) - return err + return fmt.Errorf("start tiller subprocess: %v", err) } // Wait for success of "helm version" + helmCounter := 0 for { cliHelm := &CliHelm{} stdout, stderr, err := cliHelm.Cmd("version") - log.Debugf("helm version: %s %s", stdout, stderr) + if err != nil { - log.Errorf("unable to get helm version: %v\n%v %v", err, stdout, stderr) - time.Sleep(100*time.Millisecond) + // log stdout and stderr as fields. Json formatter will escape multilines. + logEntry.WithField("stdout", stdout). + WithField("stderr", stderr). + Warnf("Unable to get tiller version: %v", err) + time.Sleep(250*time.Millisecond) } else { - log.Infof("tiller started and is available") + logEntry.WithField("stdout", stdout). + WithField("stderr", stderr). + Debugf("Output of helm version") + logEntry.Infof("Tiller started and is available") break } + helmCounter += 1 + if helmCounter > MaxHelmVersionWaits { + return fmt.Errorf("wait tiller timeout") + } } go func() { err = tillerCmd.Wait() if err != nil { - log.Errorf("Tiller process exited, now stop. (%v)", err) + logEntry.Errorf("Tiller process exited, now stop. Wait error: %v", err) } else { - log.Errorf("Tiller process exited, now stop.") + logEntry.Errorf("Tiller process exited, now stop.") } os.Exit(1) }() @@ -85,7 +98,7 @@ func TillerHealthHandler(tillerProbeAddress string, tillerProbePort int32) func( return } - body, err := ioutil.ReadAll(res.Body) + tillerLivenessBody, err := ioutil.ReadAll(res.Body) res.Body.Close() if err != nil { writer.WriteHeader(http.StatusInternalServerError) @@ -93,8 +106,7 @@ func TillerHealthHandler(tillerProbeAddress string, tillerProbePort int32) func( return } - // TODO translate response of GET /liveness from tiller probe port writer.WriteHeader(http.StatusOK) - writer.Write(body) + writer.Write(tillerLivenessBody) } } diff --git a/pkg/module_manager/global_hook.go b/pkg/module_manager/global_hook.go index 4e516729..74c7d6a6 100644 --- a/pkg/module_manager/global_hook.go +++ b/pkg/module_manager/global_hook.go @@ -91,9 +91,7 @@ func (h *GlobalHook) handleGlobalValuesPatch(currentValues utils.Values, valuesP return result, nil } -func (h *GlobalHook) run(bindingType BindingType, context []BindingContext) error { - log.Infof("Running global hook '%s' binding '%s' ...", h.Name, bindingType) - +func (h *GlobalHook) Run(bindingType BindingType, context []BindingContext, logLabels map[string]string) error { // Convert context for version versionedContext := make([]interface{}, 0, len(context)) for _, c := range context { @@ -101,6 +99,7 @@ func (h *GlobalHook) run(bindingType BindingType, context []BindingContext) erro } globalHookExecutor := NewHookExecutor(h, versionedContext) + globalHookExecutor.WithLogLabels(logLabels) patches, err := globalHookExecutor.Run() if err != nil { return fmt.Errorf("global hook '%s' failed: %s", h.Name, err) diff --git a/pkg/module_manager/hook.go b/pkg/module_manager/hook.go index 65896f49..bc690758 100644 --- a/pkg/module_manager/hook.go +++ b/pkg/module_manager/hook.go @@ -6,6 +6,7 @@ import ( "path/filepath" "sort" + "github.com/flant/addon-operator/pkg/utils" "github.com/kennygrant/sanitize" log "github.com/sirupsen/logrus" @@ -46,10 +47,8 @@ func (h *CommonHook) GetPath() string { return h.Path } - +// SearchGlobalHooks recursively find all executables in hooksDir. Absent hooksDir is not an error. func SearchGlobalHooks(hooksDir string) (hooks []*GlobalHook, err error) { - log.Debug("INIT: search global hooks...") - if _, err := os.Stat(hooksDir); os.IsNotExist(err) { return nil, nil } @@ -71,8 +70,6 @@ func SearchGlobalHooks(hooksDir string) (hooks []*GlobalHook, err error) { return nil, err } - log.Infof("INIT: global hook '%s'", hookName) - globalHook := NewGlobalHook(hookName, hookPath) hooks = append(hooks, globalHook) @@ -82,8 +79,6 @@ func SearchGlobalHooks(hooksDir string) (hooks []*GlobalHook, err error) { } func SearchModuleHooks(module *Module) (hooks []*ModuleHook, err error) { - log.Infof("INIT: module '%s' hooks ...", module.Name) - hooksDir := filepath.Join(module.Path, "hooks") if _, err := os.Stat(hooksDir); os.IsNotExist(err) { return nil, nil @@ -106,8 +101,6 @@ func SearchModuleHooks(module *Module) (hooks []*ModuleHook, err error) { return nil, err } - log.Infof("INIT: hook '%s' ...", hookName) - moduleHook := NewModuleHook(hookName, hookPath) moduleHook.WithModule(module) @@ -118,41 +111,8 @@ func SearchModuleHooks(module *Module) (hooks []*ModuleHook, err error) { } -func LoadModuleHooksConfig(hooks []*ModuleHook) error { - for _, hook := range hooks { - configOutput, err := NewHookExecutor(hook, nil).Config() - if err != nil { - return fmt.Errorf("hook '%s' config: %s", hook.GetPath(), err) - } - - err = hook.WithConfig(configOutput) - if err != nil { - return fmt.Errorf("creating global hook '%s': %s", hook.GetName(), err) - } - } - - return nil -} - -func LoadGlobalHooksConfig(hooks []*GlobalHook) error { - for _, hook := range hooks { - configOutput, err := NewHookExecutor(hook, nil).Config() - if err != nil { - return fmt.Errorf("hook '%s' config: %s", hook.GetPath(), err) - } - - err = hook.WithConfig(configOutput) - if err != nil { - return fmt.Errorf("creating module hook '%s': %s", hook.GetName(), err) - } - } - - return nil -} - - func (mm *MainModuleManager) RegisterGlobalHooks() error { - log.Debug("INIT: global hooks") + log.Debug("Search and register global hooks") mm.globalHooksOrder = make(map[BindingType][]*GlobalHook) mm.globalHooksByName = make(map[string]*GlobalHook) @@ -161,43 +121,69 @@ func (mm *MainModuleManager) RegisterGlobalHooks() error { if err != nil { return err } - - err = LoadGlobalHooksConfig(hooks) - if err != nil { - return err - } + log.Debug("Found %d global hooks", len(hooks)) for _, globalHook := range hooks { + logEntry := log.WithField("hook", globalHook.Name). + WithField("hook.type", "global") + + configOutput, err := NewHookExecutor(globalHook, nil).Config() + if err != nil { + logEntry.Errorf("Run --config: %s", err) + return fmt.Errorf("global hook --config run problem") + } + + err = globalHook.WithConfig(configOutput) + if err != nil { + logEntry.Errorf("Hook return bad config: %s", err) + return fmt.Errorf("global hook return bad config") + } + globalHook.WithModuleManager(mm) // register global hook in indexes for _, binding := range globalHook.Config.Bindings() { mm.globalHooksOrder[binding] = append(mm.globalHooksOrder[binding], globalHook) } mm.globalHooksByName[globalHook.Name] = globalHook + + logEntry.Infof("Registered") } return nil } -func (mm *MainModuleManager) RegisterModuleHooks(module *Module) error { +func (mm *MainModuleManager) RegisterModuleHooks(module *Module, logLabels map[string]string) error { + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)).WithField("module", module.Name) + if _, ok := mm.modulesHooksOrderByName[module.Name]; ok { - log.Debugf("INIT: module '%s' hooks: already initialized", module.Name) + logEntry.Debugf("Module hooks already registered") return nil } - log.Infof("INIT: module '%s' hooks ...", module.Name) + logEntry.Debugf("Search and register hooks") hooks, err := SearchModuleHooks(module) if err != nil { return err } - - err = LoadModuleHooksConfig(hooks) - if err != nil { - return err - } + logEntry.Debugf("Found %d hooks", len(hooks)) for _, moduleHook := range hooks { + hookLogEntry := logEntry.WithField("hook", moduleHook.Name). + WithField("hook.type", "module") + + configOutput, err := NewHookExecutor(moduleHook, nil).Config() + if err != nil { + hookLogEntry.Errorf("Run --config: %s", err) + return fmt.Errorf("module hook --config run problem") + } + + err = moduleHook.WithConfig(configOutput) + if err != nil { + hookLogEntry.Errorf("Hook return bad config: %s", err) + return fmt.Errorf("module hook return bad config") + } + moduleHook.WithModuleManager(mm) // register module hook in indexes for _, binding := range moduleHook.Config.Bindings() { @@ -206,6 +192,8 @@ func (mm *MainModuleManager) RegisterModuleHooks(module *Module) error { } mm.modulesHooksOrderByName[module.Name][binding] = append(mm.modulesHooksOrderByName[module.Name][binding], moduleHook) } + + hookLogEntry.Infof("Registered") } return nil diff --git a/pkg/module_manager/hook/kube_event/hooks_controller.go b/pkg/module_manager/hook/kube_event/hooks_controller.go index dc23de23..202d9e24 100644 --- a/pkg/module_manager/hook/kube_event/hooks_controller.go +++ b/pkg/module_manager/hook/kube_event/hooks_controller.go @@ -11,15 +11,17 @@ import ( "github.com/flant/addon-operator/pkg/module_manager" "github.com/flant/addon-operator/pkg/task" + "github.com/flant/addon-operator/pkg/utils" ) + type KubernetesHooksController interface { WithModuleManager(moduleManager module_manager.ModuleManager) WithKubeEventsManager(kube_events_manager.KubeEventsManager) - EnableGlobalHooks() error - EnableModuleHooks(moduleName string) error - DisableModuleHooks(moduleName string) error - HandleEvent(kubeEvent kube_events_manager.KubeEvent) ([]task.Task, error) + EnableGlobalHooks(logLabels map[string]string) error + EnableModuleHooks(moduleName string, logLabels map[string]string) error + DisableModuleHooks(moduleName string, logLabels map[string]string) error + HandleEvent(kubeEvent kube_events_manager.KubeEvent, logLabels map[string]string) ([]task.Task, error) } type kubernetesHooksController struct { @@ -54,14 +56,16 @@ func (c *kubernetesHooksController) WithKubeEventsManager(kem kube_events_manage } // EnableGlobalHooks starts kube events informers for all global hooks -func (c *kubernetesHooksController) EnableGlobalHooks() error { +func (c *kubernetesHooksController) EnableGlobalHooks(logLabels map[string]string) error { globalHooks := c.moduleManager.GetGlobalHooksInOrder(module_manager.KubeEvents) for _, globalHookName := range globalHooks { globalHook, _ := c.moduleManager.GetGlobalHook(globalHookName) for _, config := range globalHook.Config.OnKubernetesEvents { - logEntry := log.WithField("hook", globalHook.Name).WithField("hook.type", "global") + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)). + WithField("hook", globalHook.Name). + WithField("hook.type", "global") err := c.kubeEventsManager.AddMonitor("", config.Monitor, logEntry) if err != nil { return fmt.Errorf("run kube monitor for hook %s: %s", globalHook.Name, err) @@ -74,11 +78,14 @@ func (c *kubernetesHooksController) EnableGlobalHooks() error { } } + // Start created informers + c.kubeEventsManager.Start() + return nil } // EnableModuleHooks starts kube events informers for all module hooks -func (c *kubernetesHooksController) EnableModuleHooks(moduleName string) error { +func (c *kubernetesHooksController) EnableModuleHooks(moduleName string, logLabels map[string]string) error { for _, enabledModuleName := range c.EnabledModules { if enabledModuleName == moduleName { // already enabled @@ -95,7 +102,10 @@ func (c *kubernetesHooksController) EnableModuleHooks(moduleName string) error { moduleHook, _ := c.moduleManager.GetModuleHook(moduleHookName) for _, config := range moduleHook.Config.OnKubernetesEvents { - logEntry := log.WithField("hook", moduleHook.Name).WithField("hook.type", "module").WithField("module", moduleHook.Module.Name) + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)). + WithField("hook", moduleHook.Name). + WithField("hook.type", "module"). + WithField("module", moduleHook.Module.Name) err := c.kubeEventsManager.AddMonitor("", config.Monitor, logEntry) if err != nil { return fmt.Errorf("run kube monitor for hook %s: %s", moduleHook.Name, err) @@ -110,13 +120,13 @@ func (c *kubernetesHooksController) EnableModuleHooks(moduleName string) error { c.EnabledModules = append(c.EnabledModules, moduleName) - // Start informers for new monitors + // Start created informers c.kubeEventsManager.Start() return nil } // DisableModuleHooks stops all monitors for all hooks in module -func (c *kubernetesHooksController) DisableModuleHooks(moduleName string) error { +func (c *kubernetesHooksController) DisableModuleHooks(moduleName string, logLabels map[string]string) error { // TODO remove EnabledModules index. ConfigId is now in moduleHook.Config.OnKubernetesEvents[].Monitor.Metadata.ConfigId // loop through module hooks and check if configId is in c.ModuleHooks, stop monitor and delete a map item. moduleEnabledInd := -1 @@ -156,23 +166,29 @@ func (c *kubernetesHooksController) DisableModuleHooks(moduleName string) error } // HandleEvent creates a task from kube event -func (c *kubernetesHooksController) HandleEvent(kubeEvent kube_events_manager.KubeEvent) ([]task.Task, error) { +func (c *kubernetesHooksController) HandleEvent(kubeEvent kube_events_manager.KubeEvent, logLabels map[string]string) ([]task.Task, error) { res := make([]task.Task, 0) globalHook, hasGlobalHook := c.GlobalHooks[kubeEvent.ConfigId] moduleHook, hasModuleHook := c.ModuleHooks[kubeEvent.ConfigId] if !hasGlobalHook && !hasModuleHook { - return nil, fmt.Errorf("Possible a bug: kubernets event '%s/%s/%s %s' is received, but no hook is found", kubeEvent.Namespace, kubeEvent.Kind, kubeEvent.Name, kubeEvent.Event) + return nil, fmt.Errorf("Possible a bug: kubernetes event '%s/%s/%s %s' is received, but no hook is found", kubeEvent.Namespace, kubeEvent.Kind, kubeEvent.Name, kubeEvent.Event) } + hookLabels := utils.MergeLabels(logLabels) + var taskType task.TaskType var kubeHook *kube_event.KubeEventHook if hasGlobalHook { kubeHook = globalHook taskType = task.GlobalHookRun + hookLabels["hook"] = globalHook.HookName + hookLabels["hook.type"] = "global" } else { kubeHook = moduleHook taskType = task.ModuleHookRun + hookLabels["hook"] = moduleHook.HookName + hookLabels["hook.type"] = "module" } switch kubeEvent.Type { @@ -194,7 +210,8 @@ func (c *kubernetesHooksController) HandleEvent(kubeEvent kube_events_manager.Ku newTask := task.NewTask(taskType, kubeHook.HookName). WithBinding(module_manager.KubeEvents). WithBindingContext(bindingContext). - WithAllowFailure(kubeHook.AllowFailure) + WithAllowFailure(kubeHook.AllowFailure). + WithLogLabels(hookLabels) res = append(res, newTask) default: @@ -219,49 +236,11 @@ func (c *kubernetesHooksController) HandleEvent(kubeEvent kube_events_manager.Ku newTask := task.NewTask(taskType, kubeHook.HookName). WithBinding(module_manager.KubeEvents). WithBindingContext(bindingContext). - WithAllowFailure(kubeHook.AllowFailure) + WithAllowFailure(kubeHook.AllowFailure). + WithLogLabels(hookLabels) res = append(res, newTask) } - - //var desc *kube_event.KubeEventHook - //var taskType task.TaskType - - //if moduleDesc, hasKey := c.ModuleHooks[kubeEvent.ConfigId]; hasKey { - // desc = moduleDesc - // taskType = task.ModuleHookRun - //} else if globalDesc, hasKey := c.GlobalHooks[kubeEvent.ConfigId]; hasKey { - // desc = globalDesc - // taskType = task.GlobalHookRun - //} - - //if desc != nil && taskType != "" { - // bindingName := desc.Name - // if desc.Name == "" { - // bindingName = module_manager.ContextBindingType[module_manager.KubeEvents] - // } - // - // bindingContext := make([]module_manager.BindingContext, 0) - // for _, kEvent := range kubeEvent.Events { - // bindingContext = append(bindingContext, module_manager.BindingContext{ - // Binding: bindingName, - // ResourceEvent: kEvent, - // ResourceNamespace: kubeEvent.Namespace, - // ResourceKind: kubeEvent.Kind, - // ResourceName: kubeEvent.Name, - // }) - // } - // - // newTask := task.NewTask(taskType, desc.HookName). - // WithBinding(module_manager.KubeEvents). - // WithBindingContext(bindingContext). - // WithAllowFailure(desc.Config.AllowFailure) - // - // res = append(res, newTask) - //} else { - // return nil, fmt.Errorf("Unknown kube event: no such config id '%s' registered", kubeEvent.ConfigId) - //} - return res, nil } diff --git a/pkg/module_manager/hook/schedule/hooks_controller.go b/pkg/module_manager/hook/schedule/hooks_controller.go index 652e59ae..1fea0d36 100644 --- a/pkg/module_manager/hook/schedule/hooks_controller.go +++ b/pkg/module_manager/hook/schedule/hooks_controller.go @@ -3,6 +3,7 @@ package schedule import ( "fmt" + "github.com/flant/addon-operator/pkg/utils" log "github.com/sirupsen/logrus" "github.com/flant/shell-operator/pkg/hook" @@ -19,7 +20,7 @@ type ScheduleHooksController interface { UpdateScheduleHooks() - HandleEvent(crontab string) ([]task.Task, error) + HandleEvent(crontab string, logLabels map[string]string) ([]task.Task, error) } type scheduleHooksController struct { @@ -112,7 +113,7 @@ func (c *scheduleHooksController) UpdateScheduleHooks() { c.ModuleHooks = newModuleHooks } -func (c *scheduleHooksController) HandleEvent(crontab string) ([]task.Task, error) { +func (c *scheduleHooksController) HandleEvent(crontab string, logLabels map[string]string) ([]task.Task, error) { res := make([]task.Task, 0) // search for global hooks by crontab @@ -124,10 +125,14 @@ func (c *scheduleHooksController) HandleEvent(crontab string) ([]task.Task, erro log.Errorf("Possible a bug: global hook '%s' is registered for schedule but not found", scheduleHook.HookName) continue } + hookLabels := utils.MergeLabels(logLabels) + hookLabels["hook"] = scheduleHook.HookName + hookLabels["hook.type"] = "global" newTask := task.NewTask(task.GlobalHookRun, scheduleHook.HookName). WithBinding(module_manager.Schedule). AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: scheduleHook.ConfigName}}). - WithAllowFailure(scheduleHook.AllowFailure) + WithAllowFailure(scheduleHook.AllowFailure). + WithLogLabels(hookLabels) res = append(res, newTask) } @@ -135,15 +140,20 @@ func (c *scheduleHooksController) HandleEvent(crontab string) ([]task.Task, erro scheduleModuleHooks := c.ModuleHooks.GetHooksForSchedule(crontab) for _, scheduleHook := range scheduleModuleHooks { - _, err := c.moduleManager.GetModuleHook(scheduleHook.HookName) + moduleHook, err := c.moduleManager.GetModuleHook(scheduleHook.HookName) if err != nil { log.Errorf("Possible a bug: module hook '%s' is registered for schedule but not found", scheduleHook.HookName) continue } + hookLabels := utils.MergeLabels(logLabels) + hookLabels["module"] = moduleHook.Module.Name + hookLabels["hook"] = scheduleHook.HookName + hookLabels["hook.type"] = "module" newTask := task.NewTask(task.ModuleHookRun, scheduleHook.HookName). WithBinding(module_manager.Schedule). AppendBindingContext(module_manager.BindingContext{BindingContext: hook.BindingContext{Binding: scheduleHook.ConfigName}}). - WithAllowFailure(scheduleHook.AllowFailure) + WithAllowFailure(scheduleHook.AllowFailure). + WithLogLabels(hookLabels) res = append(res, newTask) } diff --git a/pkg/module_manager/hook_executor.go b/pkg/module_manager/hook_executor.go index 3ab68831..1c5c06f3 100644 --- a/pkg/module_manager/hook_executor.go +++ b/pkg/module_manager/hook_executor.go @@ -21,15 +21,21 @@ type HookExecutor struct { ContextPath string ConfigValuesPatchPath string ValuesPatchPath string + LogLabels map[string]string } func NewHookExecutor(h Hook, context interface{}) *HookExecutor { return &HookExecutor{ Hook: h, Context: context, + LogLabels: map[string]string{}, } } +func (e *HookExecutor) WithLogLabels(logLabels map[string]string) { + e.LogLabels = logLabels +} + func (e *HookExecutor) Run() (patches map[utils.ValuesPatchType]*utils.ValuesPatch, err error) { patches = make(map[utils.ValuesPatchType]*utils.ValuesPatch) @@ -45,23 +51,23 @@ func (e *HookExecutor) Run() (patches map[utils.ValuesPatchType]*utils.ValuesPat for envName, filePath := range tmpFiles { envs = append(envs, fmt.Sprintf("%s=%s", envName, filePath)) } - envs = append(envs, helm.NewHelmCli(nil).CommandEnv()...) + envs = append(envs, helm.NewHelmCli(log.NewEntry(log.StandardLogger())).CommandEnv()...) cmd := executor.MakeCommand("", e.Hook.GetPath(), []string{}, envs) - err = executor.Run(cmd) + err = executor.RunAndLogLines(cmd, e.LogLabels) if err != nil { - return nil, fmt.Errorf("%s FAILED: %s", e.Hook.GetName(), err) + return nil, err } patches[utils.ConfigMapPatch], err = utils.ValuesPatchFromFile(e.ConfigValuesPatchPath) if err != nil { - return nil, fmt.Errorf("got bad config values json patch from hook %s: %s", e.Hook.GetName(), err) + return nil, fmt.Errorf("got bad json patch for config values: %s", err) } patches[utils.MemoryValuesPatch], err = utils.ValuesPatchFromFile(e.ValuesPatchPath) if err != nil { - return nil, fmt.Errorf("got bad values json patch from hook %s: %s", e.Hook.GetName(), err) + return nil, fmt.Errorf("got bad json patch for values: %s", err) } return patches, nil @@ -70,7 +76,7 @@ func (e *HookExecutor) Run() (patches map[utils.ValuesPatchType]*utils.ValuesPat func (e *HookExecutor) Config() (configOutput []byte, err error) { envs := []string{} envs = append(envs, os.Environ()...) - envs = append(envs, helm.NewHelmCli(nil).CommandEnv()...) + envs = append(envs, helm.NewHelmCli(log.NewEntry(log.StandardLogger())).CommandEnv()...) cmd := executor.MakeCommand("", e.Hook.GetPath(), []string{"--config"}, envs) @@ -79,8 +85,8 @@ func (e *HookExecutor) Config() (configOutput []byte, err error) { output, err := executor.Output(cmd) if err != nil { - log.Errorf("Hook '%s' config failed: %v, output:\n%s", e.Hook.GetName(), err, string(output)) - return nil, fmt.Errorf("%s FAILED: %s", e.Hook.GetName(), err) + log.Debugf("Hook '%s' config failed: %v, output:\n%s", e.Hook.GetName(), err, string(output)) + return nil, err } log.Debugf("Hook '%s' config output:\n%s", e.Hook.GetName(), string(output)) diff --git a/pkg/module_manager/module.go b/pkg/module_manager/module.go index 1cccec46..c21b00e1 100644 --- a/pkg/module_manager/module.go +++ b/pkg/module_manager/module.go @@ -24,7 +24,6 @@ import ( type Module struct { Name string - DirectoryName string Path string // module values from modules/values.yaml file CommonStaticConfig *utils.ModuleConfig @@ -34,10 +33,15 @@ type Module struct { moduleManager *MainModuleManager } -func NewModule(mm *MainModuleManager) *Module { - module := &Module{} - module.moduleManager = mm - return module +func NewModule(name, path string) *Module { + return &Module{ + Name: name, + Path: path, + } +} + +func (m *Module) WithModuleManager(moduleManager *MainModuleManager) { + m.moduleManager = moduleManager } func (m *Module) SafeName() string { @@ -46,18 +50,18 @@ func (m *Module) SafeName() string { // Run is a phase of module lifecycle that runs onStartup and beforeHelm hooks, helm upgrade --install command and afterHelm hook. // It is a handler of task MODULE_RUN -func (m *Module) Run(onStartup bool) error { +func (m *Module) Run(onStartup bool, logLabels map[string]string) error { if err := m.cleanup(); err != nil { return err } if onStartup { - if err := m.runHooksByBinding(OnStartup); err != nil { + if err := m.runHooksByBinding(OnStartup, logLabels); err != nil { return err } } - if err := m.runHooksByBinding(BeforeHelm); err != nil { + if err := m.runHooksByBinding(BeforeHelm, logLabels); err != nil { return err } @@ -65,7 +69,7 @@ func (m *Module) Run(onStartup bool) error { return err } - if err := m.runHooksByBinding(AfterHelm); err != nil { + if err := m.runHooksByBinding(AfterHelm, logLabels); err != nil { return err } @@ -75,8 +79,10 @@ func (m *Module) Run(onStartup bool) error { // TODO LOG: add field 'on startup' // Delete removes helm release if it exists and runs afterDeleteHelm hooks. // It is a handler for MODULE_DELETE task. -func (m *Module) Delete() error { - logEntry := log.WithField("module", m.Name).WithField("phase", "delete") +func (m *Module) Delete(logLabels map[string]string) error { + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)). + WithField("module", m.Name). + WithField("phase", "delete") // Если есть chart, но нет релиза — warning // если нет чарта — молча перейти к хукам @@ -99,7 +105,7 @@ func (m *Module) Delete() error { } } - return m.runHooksByBinding(AfterDeleteHelm) + return m.runHooksByBinding(AfterDeleteHelm, logLabels) } @@ -224,7 +230,7 @@ func (m *Module) runHelmInstall() error { } -func (m *Module) runHooksByBinding(binding BindingType) error { +func (m *Module) runHooksByBinding(binding BindingType, logLabels map[string]string) error { moduleHooksAfterHelm, err := m.moduleManager.GetModuleHooksInOrder(m.Name, binding) if err != nil { return err @@ -236,13 +242,13 @@ func (m *Module) runHooksByBinding(binding BindingType) error { return err } - err = moduleHook.run(binding, []BindingContext{ + err = moduleHook.Run(binding, []BindingContext{ { BindingContext: hook2.BindingContext{ Binding: ContextBindingType[binding], }, }, - }) + }, logLabels) if err != nil { return err } @@ -436,12 +442,13 @@ func (m *Module) readModuleEnabledResult(filePath string) (bool, error) { return false, fmt.Errorf("expected 'true' or 'false', got '%s'", value) } -func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string) (bool, error) { +func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string, logLabels map[string]string) (bool, error) { + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) enabledScriptPath := filepath.Join(m.Path, "enabled") f, err := os.Stat(enabledScriptPath) if os.IsNotExist(err) { - log.Debugf("MODULE '%s': ENABLED. Enabled script is not exist!", m.Name) + logEntry.Debugf("MODULE '%s' is ENABLED. Enabled script is not exist!", m.Name) return true, nil } else if err != nil { return false, err @@ -465,8 +472,7 @@ func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string) (bool, if err != nil { return false, err } - - log.Infof("MODULE '%s': run enabled script '%s'...", m.Name, enabledScriptPath) + logEntry.Debugf("Execute enabled script '%s', preceding modules: %v", enabledScriptPath, precedingEnabledModules) envs := make([]string, 0) envs = append(envs, os.Environ()...) @@ -485,66 +491,79 @@ func (m *Module) checkIsEnabledByScript(precedingEnabledModules []string) (bool, return false, fmt.Errorf("bad enabled result in file MODULE_ENABLED_RESULT=\"%s\" from enabled script '%s' for module '%s': %s", enabledResultFilePath, enabledScriptPath, m.Name, err) } + result := "Disabled" if moduleEnabled { - log.Debugf("Module '%s' ENABLED with script. Preceding: %s", m.Name, precedingEnabledModules) - return true, nil + result = "Enabled" + } + logEntry.Infof("Enabled script run successful, result: module %q", result) + return moduleEnabled, nil +} + +var ValidModuleNameRe = regexp.MustCompile(`^[0-9][0-9][0-9]-(.*)$`) + +func SearchModules(modulesDir string) (modules []*Module, err error) { + files, err := ioutil.ReadDir(modulesDir) // returns a list of modules sorted by filename + if err != nil { + return nil, fmt.Errorf("list modules directory '%s': %s", modulesDir, err) } - log.Debugf("Module '%s' DISABLED with script. Preceding: %s ", m.Name, precedingEnabledModules) - return false, nil + badModulesDirs := make([]string, 0) + modules = make([]*Module, 0) + + for _, file := range files { + if !file.IsDir() { + continue + } + matchRes := ValidModuleNameRe.FindStringSubmatch(file.Name()) + if matchRes != nil { + moduleName := matchRes[1] + modulePath := filepath.Join(modulesDir, file.Name()) + module := NewModule(moduleName, modulePath) + modules = append(modules, module) + } else { + badModulesDirs = append(badModulesDirs, filepath.Join(modulesDir, file.Name())) + } + } + + if len(badModulesDirs) > 0 { + return nil, fmt.Errorf("modules directory contains directories not matched ValidModuleRegex '%s': %s", ValidModuleNameRe, strings.Join(badModulesDirs, ", ")) + } + + return } -// initModulesIndex load all available modules from modules directory +// RegisterModules load all available modules from modules directory // FIXME: Only 000-name modules are loaded, allow non-prefixed modules. -func (mm *MainModuleManager) initModulesIndex() error { - log.Debug("INIT: Search modules ...") +func (mm *MainModuleManager) RegisterModules() error { + log.Debug("Search and register modules") - files, err := ioutil.ReadDir(mm.ModulesDir) // returns a list of modules sorted by filename + modules, err := SearchModules(mm.ModulesDir) if err != nil { - return fmt.Errorf("INIT: cannot list modules directory '%s': %s", mm.ModulesDir, err) + return err } + log.Debug("Found %d modules", len(modules)) // load global and modules common static values from modules/values.yaml if err := mm.loadCommonStaticValues(); err != nil { - return fmt.Errorf("INIT: load common values: %s", err) + return fmt.Errorf("load common values for modules: %s", err) } - var validModuleName = regexp.MustCompile(`^[0-9][0-9][0-9]-(.*)$`) + for _, module := range modules { + logEntry := log.WithField("module", module.Name) - badModulesDirs := make([]string, 0) + module.WithModuleManager(mm) - for _, file := range files { - if file.IsDir() { - matchRes := validModuleName.FindStringSubmatch(file.Name()) - if matchRes != nil { - moduleName := matchRes[1] - log.Infof("INIT: Register module '%s'", moduleName) - - modulePath := filepath.Join(mm.ModulesDir, file.Name()) - - module := NewModule(mm) - module.Name = moduleName - module.DirectoryName = file.Name() - module.Path = modulePath - - // load static config from values.yaml - err := module.loadStaticValues() - if err != nil { - return err - } - - mm.allModulesByName[module.Name] = module - mm.allModulesNamesInOrder = append(mm.allModulesNamesInOrder, module.Name) - } else { - badModulesDirs = append(badModulesDirs, filepath.Join(mm.ModulesDir, file.Name())) - } + // load static config from values.yaml + err := module.loadStaticValues() + if err != nil { + logEntry.Errorf("Load values.yaml: %s", err) + return fmt.Errorf("bad module values") } - } - log.Debugf("INIT: initModulesIndex registered modules: %v", mm.allModulesByName) + mm.allModulesByName[module.Name] = module + mm.allModulesNamesInOrder = append(mm.allModulesNamesInOrder, module.Name) - if len(badModulesDirs) > 0 { - return fmt.Errorf("found directories not matched regex '%s': %s", validModuleName, strings.Join(badModulesDirs, ", ")) + logEntry.Infof("Registered") } return nil diff --git a/pkg/module_manager/module_hook.go b/pkg/module_manager/module_hook.go index 909e4b88..afbd6f72 100644 --- a/pkg/module_manager/module_hook.go +++ b/pkg/module_manager/module_hook.go @@ -99,9 +99,8 @@ func (h *ModuleHook) handleModuleValuesPatch(currentValues utils.Values, valuesP return result, nil } -func (h *ModuleHook) run(bindingType BindingType, context []BindingContext) error { +func (h *ModuleHook) Run(bindingType BindingType, context []BindingContext, logLabels map[string]string) error { moduleName := h.Module.Name - log.Infof("Running module hook '%s' binding '%s' ...", h.Name, bindingType) // Convert context for version versionedContext := make([]interface{}, 0, len(context)) @@ -110,6 +109,7 @@ func (h *ModuleHook) run(bindingType BindingType, context []BindingContext) erro } moduleHookExecutor := NewHookExecutor(h, versionedContext) + moduleHookExecutor.WithLogLabels(logLabels) patches, err := moduleHookExecutor.Run() if err != nil { return fmt.Errorf("module hook '%s' failed: %s", h.Name, err) diff --git a/pkg/module_manager/module_manager.go b/pkg/module_manager/module_manager.go index 7117e7b1..68694d66 100644 --- a/pkg/module_manager/module_manager.go +++ b/pkg/module_manager/module_manager.go @@ -6,12 +6,11 @@ import ( "sort" "strings" + "github.com/flant/addon-operator/pkg/app" log "github.com/sirupsen/logrus" hook_config "github.com/flant/shell-operator/pkg/hook" - hook_config "github.com/flant/shell-operator/pkg/hook" - "github.com/flant/addon-operator/pkg/helm" "github.com/flant/addon-operator/pkg/kube_config_manager" "github.com/flant/addon-operator/pkg/utils" @@ -22,18 +21,18 @@ import ( type ModuleManager interface { Init() error Run() - DiscoverModulesState() (*ModulesState, error) + DiscoverModulesState(logLabels map[string]string) (*ModulesState, error) GetModule(name string) (*Module, error) GetModuleNamesInOrder() []string GetGlobalHook(name string) (*GlobalHook, error) GetModuleHook(name string) (*ModuleHook, error) GetGlobalHooksInOrder(bindingType BindingType) []string GetModuleHooksInOrder(moduleName string, bindingType BindingType) ([]string, error) - DeleteModule(moduleName string) error - RunModule(moduleName string, onStartup bool) error - RunGlobalHook(hookName string, binding BindingType, bindingContext []BindingContext) error - RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext) error - RegisterModuleHooks(module *Module) error + DeleteModule(moduleName string, logLabels map[string]string) error + RunModule(moduleName string, onStartup bool, logLabels map[string]string) error + RunGlobalHook(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) error + RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) error + RegisterModuleHooks(module *Module, logLabels map[string]string) error Retry() WithDirectories(modulesDir string, globalHooksDir string, tempDir string) ModuleManager WithKubeConfigManager(kubeConfigManager kube_config_manager.KubeConfigManager) ModuleManager @@ -116,6 +115,8 @@ type MainModuleManager struct { retryOnAmbigous chan bool } +var _ ModuleManager = &MainModuleManager{} + var ( EventCh chan Event ) @@ -225,15 +226,16 @@ func NewMainModuleManager() *MainModuleManager { } } -// determineEnableStateWithScript runs enable script for each module that is enabled by config. +// RunModulesEnabledScript runs enable script for each module that is enabled by config. // Enable script receives a list of previously enabled modules. -func (mm *MainModuleManager) determineEnableStateWithScript(enabledByConfig []string) ([]string, error) { - log.Debugf("Run enable scripts for modules: %+v", enabledByConfig) +func (mm *MainModuleManager) RunModulesEnabledScript(enabledByConfig []string, logLabels map[string]string) ([]string, error) { enabledModules := make([]string, 0) for _, name := range utils.SortByReference(enabledByConfig, mm.allModulesNamesInOrder) { + moduleLogLabels := utils.MergeLabels(logLabels) + moduleLogLabels["module"] = name module := mm.allModulesByName[name] - moduleIsEnabled, err := module.checkIsEnabledByScript(enabledModules) + moduleIsEnabled, err := module.checkIsEnabledByScript(enabledModules, moduleLogLabels) if err != nil { return nil, err } @@ -243,7 +245,6 @@ func (mm *MainModuleManager) determineEnableStateWithScript(enabledByConfig []st } } - log.Debugf("Modules enabled with script: %+v", enabledModules) return enabledModules, nil } @@ -291,8 +292,10 @@ func (mm *MainModuleManager) handleNewKubeConfig(newConfig kube_config_manager.C } func (mm *MainModuleManager) handleNewKubeModuleConfigs(moduleConfigs kube_config_manager.ModuleConfigs) (*kubeUpdate, error) { - logEntry := log.WithField("operator.component", "ModuleManager"). - WithField("operator.action", "handleNewKubeModuleConfigs") + logLabels := map[string]string{ + "operator.component": "HandleConfigMap", + } + logEntry := log.WithFields(utils.LabelsToLogFields(logLabels)) logEntry.Debugf("handle changes in module sections") @@ -328,12 +331,12 @@ func (mm *MainModuleManager) handleNewKubeModuleConfigs(moduleConfigs kube_confi res.EnabledModulesByConfig = utils.SortByReference(res.EnabledModulesByConfig, mm.allModulesNamesInOrder) // Run enable scripts - logEntry.Infof("run `enabled` for %s", res.EnabledModulesByConfig) - enabledModules, err := mm.determineEnableStateWithScript(res.EnabledModulesByConfig) + logEntry.Debugf("Run enabled script for %+v", res.EnabledModulesByConfig) + enabledModules, err := mm.RunModulesEnabledScript(res.EnabledModulesByConfig, logLabels) if err != nil { return nil, err } - logEntry.Infof("enabled modules: %+v", enabledModules) + logEntry.Infof("Modules enabled by script: %+v", enabledModules) // Configure events if !reflect.DeepEqual(mm.enabledModulesInOrder, enabledModules) { @@ -428,13 +431,13 @@ func (mm *MainModuleManager) calculateEnabledModulesByConfig(moduleConfigs kube_ // Init — initialize module manager func (mm *MainModuleManager) Init() error { - log.Debug("INIT: MODULE_MANAGER") + log.Debug("Init ModuleManager") if err := mm.RegisterGlobalHooks(); err != nil { return err } - if err := mm.initModulesIndex(); err != nil { + if err := mm.RegisterModules(); err != nil { return err } @@ -444,11 +447,11 @@ func (mm *MainModuleManager) Init() error { var unknown []utils.ModuleConfig mm.enabledModulesByConfig, mm.kubeModulesConfigValues, unknown = mm.calculateEnabledModulesByConfig(kubeConfig.ModuleConfigs) + unknownNames := []string{} for _, config := range unknown { - log.Warnf("INIT: MODULE_MANAGER: ignore kube config for absent module: \n%s", - config.String(), - ) + unknownNames = append(unknownNames, config.ModuleName) } + log.Warnf("ConfigMap/%s has values for absent modules: %+v", app.ConfigMapName, unknownNames) return nil } @@ -457,8 +460,6 @@ func (mm *MainModuleManager) Init() error { func (mm *MainModuleManager) Run() { go mm.kubeConfigManager.Run() - - for { select { case <-mm.globalValuesChanged: @@ -534,7 +535,7 @@ func (mm *MainModuleManager) Retry() { // modules that should be disabled and modules that should be purged. // // This method requires that mm.enabledModulesByConfig and mm.kubeModulesConfigValues are updated. -func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err error) { +func (mm *MainModuleManager) DiscoverModulesState(logLabels map[string]string) (state *ModulesState, err error) { logEntry := log.WithField("operator.component", "moduleManager,discoverModulesState") logEntry.Debugf("DISCOVER state:\n"+ @@ -568,15 +569,16 @@ func (mm *MainModuleManager) DiscoverModulesState() (state *ModulesState, err er // modules finally enabled with enable script // no need to refresh mm.enabledModulesByConfig because // it is updated before in Init or in applyKubeUpdate - logEntry.Infof("run `enabled` for %s", mm.enabledModulesByConfig) - enabledModules, err := mm.determineEnableStateWithScript(mm.enabledModulesByConfig) - logEntry.Infof("enabled modules %s", enabledModules) + logEntry.Debugf("Run enabled script for %+v", mm.enabledModulesByConfig) + enabledModules, err := mm.RunModulesEnabledScript(mm.enabledModulesByConfig, logLabels) + logEntry.Infof("Modules enabled by script: %+v", enabledModules) + if err != nil { return nil, err } for _, moduleName := range enabledModules { - if err = mm.RegisterModuleHooks(mm.allModulesByName[moduleName]); err != nil { + if err = mm.RegisterModuleHooks(mm.allModulesByName[moduleName], logLabels); err != nil { return nil, err } } @@ -689,13 +691,13 @@ func (mm *MainModuleManager) GetModuleHooksInOrder(moduleName string, bindingTyp } // TODO: moduleManager.Module(modName).Delete() -func (mm *MainModuleManager) DeleteModule(moduleName string) error { +func (mm *MainModuleManager) DeleteModule(moduleName string, logLabels map[string]string) error { module, err := mm.GetModule(moduleName) if err != nil { return err } - if err := module.Delete(); err != nil { + if err := module.Delete(logLabels); err != nil { return err } @@ -706,20 +708,20 @@ func (mm *MainModuleManager) DeleteModule(moduleName string) error { } // RunModule runs beforeHelm hook, helm upgrade --install and afterHelm or afterDeleteHelm hook -func (mm *MainModuleManager) RunModule(moduleName string, onStartup bool) error { +func (mm *MainModuleManager) RunModule(moduleName string, onStartup bool, logLabels map[string]string) error { module, err := mm.GetModule(moduleName) if err != nil { return err } - if err := module.Run(onStartup); err != nil { + if err := module.Run(onStartup, logLabels); err != nil { return err } return nil } -func (mm *MainModuleManager) RunGlobalHook(hookName string, binding BindingType, bindingContext []BindingContext) error { +func (mm *MainModuleManager) RunGlobalHook(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) error { globalHook, err := mm.GetGlobalHook(hookName) if err != nil { return err @@ -730,7 +732,7 @@ func (mm *MainModuleManager) RunGlobalHook(hookName string, binding BindingType, return err } - if err := globalHook.run(binding, bindingContext); err != nil { + if err := globalHook.Run(binding, bindingContext, logLabels); err != nil { return err } @@ -749,7 +751,7 @@ func (mm *MainModuleManager) RunGlobalHook(hookName string, binding BindingType, return nil } -func (mm *MainModuleManager) RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext) error { +func (mm *MainModuleManager) RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) error { moduleHook, err := mm.GetModuleHook(hookName) if err != nil { return err @@ -760,7 +762,7 @@ func (mm *MainModuleManager) RunModuleHook(hookName string, binding BindingType, return err } - if err := moduleHook.run(binding, bindingContext); err != nil { + if err := moduleHook.Run(binding, bindingContext, logLabels); err != nil { return err } diff --git a/pkg/module_manager/module_manager_mock.go b/pkg/module_manager/module_manager_mock.go index 70764523..2424f16d 100644 --- a/pkg/module_manager/module_manager_mock.go +++ b/pkg/module_manager/module_manager_mock.go @@ -5,21 +5,21 @@ import "github.com/flant/addon-operator/pkg/kube_config_manager" type ModuleManagerMockFns struct { Init func() error Run func() - DiscoverModulesState func() (*ModulesState, error) + DiscoverModulesState func(logLabels map[string]string) (*ModulesState, error) GetModule func(name string) (*Module, error) GetModuleNamesInOrder func() []string GetGlobalHook func(name string) (*GlobalHook, error) GetModuleHook func(name string) (*ModuleHook, error) GetGlobalHooksInOrder func(bindingType BindingType) []string GetModuleHooksInOrder func(moduleName string, bindingType BindingType) ([]string, error) - DeleteModule func(moduleName string) error - RunModule func(moduleName string, onStartup bool) error - RunGlobalHook func(hookName string, binding BindingType, bindingContext []BindingContext) error - RunModuleHook func(hookName string, binding BindingType, bindingContext []BindingContext) error + DeleteModule func(moduleName string, logLabels map[string]string) error + RunModule func(moduleName string, onStartup bool, logLabels map[string]string) error + RunGlobalHook func(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) error + RunModuleHook func(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) error Retry func() WithDirectories func(modulesDir string, globalHooksDir string, tempDir string) ModuleManager WithKubeConfigManager func(kubeConfigManager kube_config_manager.KubeConfigManager) ModuleManager - RegisterModuleHooks func(*Module) error + RegisterModuleHooks func(*Module, map[string]string) error } @@ -47,9 +47,9 @@ func (m *ModuleManagerMock) Run() { panic("implement me") } -func (m *ModuleManagerMock) DiscoverModulesState() (*ModulesState, error) { +func (m *ModuleManagerMock) DiscoverModulesState(logLabels map[string]string) (*ModulesState, error) { if m.Fns.DiscoverModulesState != nil { - return m.Fns.DiscoverModulesState() + return m.Fns.DiscoverModulesState(logLabels) } panic("implement me") } @@ -96,30 +96,30 @@ func (m *ModuleManagerMock) GetModuleHooksInOrder(moduleName string, bindingType panic("implement me") } -func (m *ModuleManagerMock) DeleteModule(moduleName string) error { +func (m *ModuleManagerMock) DeleteModule(moduleName string, logLabels map[string]string) error { if m.Fns.DeleteModule != nil { - return m.Fns.DeleteModule(moduleName) + return m.Fns.DeleteModule(moduleName, logLabels) } panic("implement me") } -func (m *ModuleManagerMock) RunModule(moduleName string, onStartup bool) error { +func (m *ModuleManagerMock) RunModule(moduleName string, onStartup bool, logLabels map[string]string) error { if m.Fns.RunModule != nil { - return m.Fns.RunModule(moduleName, onStartup) + return m.Fns.RunModule(moduleName, onStartup, logLabels) } panic("implement me") } -func (m *ModuleManagerMock) RunGlobalHook(hookName string, binding BindingType, bindingContext []BindingContext) error { +func (m *ModuleManagerMock) RunGlobalHook(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) error { if m.Fns.RunGlobalHook != nil { - return m.Fns.RunGlobalHook(hookName, binding, bindingContext) + return m.Fns.RunGlobalHook(hookName, binding, bindingContext, logLabels) } panic("implement me") } -func (m *ModuleManagerMock) RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext) error { +func (m *ModuleManagerMock) RunModuleHook(hookName string, binding BindingType, bindingContext []BindingContext, logLabels map[string]string) error { if m.Fns.RunModuleHook != nil { - return m.Fns.RunModuleHook(hookName, binding, bindingContext) + return m.Fns.RunModuleHook(hookName, binding, bindingContext, logLabels) } panic("implement me") } @@ -145,9 +145,9 @@ func (m *ModuleManagerMock) WithKubeConfigManager(kubeConfigManager kube_config_ panic("implement me") } -func (m *ModuleManagerMock) RegisterModuleHooks(module *Module) error { +func (m *ModuleManagerMock) RegisterModuleHooks(module *Module, logLabels map[string]string) error { if m.Fns.RegisterModuleHooks != nil { - return m.Fns.RegisterModuleHooks(module) + return m.Fns.RegisterModuleHooks(module, logLabels) } panic("implement me") } diff --git a/pkg/module_manager/module_manager_test.go b/pkg/module_manager/module_manager_test.go index e840fcd3..b862dcdd 100644 --- a/pkg/module_manager/module_manager_test.go +++ b/pkg/module_manager/module_manager_test.go @@ -38,7 +38,7 @@ func initModuleManager(t *testing.T, mm *MainModuleManager, configPath string) { mm.WithDirectories(filepath.Join(rootDir, "modules"), filepath.Join(rootDir, "global-hooks"), tempDir) - if err := mm.initModulesIndex(); err != nil { + if err := mm.RegisterModules(); err != nil { t.Fatal(err) } @@ -241,7 +241,6 @@ func Test_MainModuleManager_Get_Module(t *testing.T) { expectedModule := &Module{ Name: "module", Path: filepath.Join(mm.ModulesDir, "000-module"), - DirectoryName: "000-module", CommonStaticConfig: &utils.ModuleConfig{ ModuleName: "module", Values: utils.Values{}, diff --git a/pkg/task/task.go b/pkg/task/task.go index 2cbb643d..58992b11 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -6,23 +6,25 @@ import ( "time" "github.com/flant/addon-operator/pkg/module_manager" + "github.com/flant/addon-operator/pkg/utils" + "gopkg.in/satori/go.uuid.v1" ) type TaskType string const ( - ModuleDelete TaskType = "TASK_MODULE_DELETE" - ModuleRun TaskType = "TASK_MODULE_RUN" - ModuleHookRun TaskType = "TASK_MODULE_HOOK_RUN" - GlobalHookRun TaskType = "TASK_GLOBAL_HOOK_RUN" - DiscoverModulesState TaskType = "TASK_DISCOVER_MODULES_STATE" + ModuleDelete TaskType = "ModuleDelete" + ModuleRun TaskType = "ModuleRun" + ModuleHookRun TaskType = "ModuleHookRun" + GlobalHookRun TaskType = "GlobalHookRun" + DiscoverModulesState TaskType = "DiscoverModulesState" // удаление релиза без сведений о модуле - ModulePurge TaskType = "TASK_MODULE_PURGE" + ModulePurge TaskType = "ModulePurge" // retry module_manager-а - ModuleManagerRetry TaskType = "TASK_MODULE_MANAGER_RETRY" + ModuleManagerRetry TaskType = "ModuleManagerRetry" // вспомогательные задачи: задержка и остановка обработки - Delay TaskType = "TASK_DELAY" - Stop TaskType = "TASK_STOP" + Delay TaskType = "Delay" + Stop TaskType = "Stop" ) type Task interface { @@ -35,6 +37,7 @@ type Task interface { GetDelay() time.Duration GetAllowFailure() bool GetOnStartupHooks() bool + GetLogLabels() map[string]string } type BaseTask struct { @@ -47,8 +50,11 @@ type BaseTask struct { AllowFailure bool // Task considered as 'ok' if hook failed. False by default. Can be true for some schedule hooks. OnStartupHooks bool // Run module onStartup hooks on Addon-operator startup or on module enabled. + LogLabels map[string]string } +var _ Task = &BaseTask{} + func NewTask(taskType TaskType, name string) *BaseTask { return &BaseTask{ FailureCount: 0, @@ -56,6 +62,7 @@ func NewTask(taskType TaskType, name string) *BaseTask { Type: taskType, AllowFailure: false, BindingContext: make([]module_manager.BindingContext, 0), + LogLabels: map[string]string{"task.id": uuid.NewV4().String()}, } } @@ -87,6 +94,10 @@ func (t *BaseTask) GetOnStartupHooks() bool { return t.OnStartupHooks } +func (t *BaseTask) GetLogLabels() map[string]string { + return t.LogLabels +} + func (t *BaseTask) WithBinding(binding module_manager.BindingType) *BaseTask { t.Binding = binding return t @@ -112,6 +123,11 @@ func (t *BaseTask) WithOnStartupHooks(onStartupHooks bool) *BaseTask { return t } +func (t *BaseTask) WithLogLabels(labels map[string]string) *BaseTask { + t.LogLabels = utils.MergeLabels(t.LogLabels, labels) + return t +} + func (t *BaseTask) DumpAsText() string { var buf bytes.Buffer buf.WriteString(fmt.Sprintf("%s '%s'", t.Type, t.Name)) diff --git a/pkg/utils/merge_labels.go b/pkg/utils/merge_labels.go new file mode 100644 index 00000000..e0a91720 --- /dev/null +++ b/pkg/utils/merge_labels.go @@ -0,0 +1,28 @@ +package utils + +import ( + log "github.com/sirupsen/logrus" +) + +// MergeLabels merges several maps into one. Last map keys overrides keys from first maps. +// +// Can be used to copy a map if just one argument is used. +func MergeLabels(labelsMaps ...map[string]string) map[string]string { + labels := make(map[string]string) + for _, labelsMap := range labelsMaps { + for k, v := range labelsMap { + labels[k] = v + } + } + return labels +} + +func LabelsToLogFields(labelsMaps ...map[string]string) log.Fields { + fields := log.Fields{} + for _, labels := range labelsMaps { + for k, v := range labels { + fields[k] = v + } + } + return fields +} From 0dbe7fe1cc2f9fbfa70dc345345336fcf41808ca Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Wed, 23 Oct 2019 19:13:33 +0300 Subject: [PATCH 5/8] feat: json logging --- cmd/addon-operator/main.go | 3 +++ go.mod | 2 +- go.sum | 1 + pkg/addon-operator/start.go | 4 ---- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/addon-operator/main.go b/cmd/addon-operator/main.go index bd5fb49a..182c19b9 100644 --- a/cmd/addon-operator/main.go +++ b/cmd/addon-operator/main.go @@ -5,6 +5,7 @@ import ( "os" "gopkg.in/alecthomas/kingpin.v2" + log "github.com/sirupsen/logrus" shell_operator_app "github.com/flant/shell-operator/pkg/app" "github.com/flant/shell-operator/pkg/executor" @@ -32,6 +33,8 @@ func main() { Default(). Action(func(c *kingpin.ParseContext) error { shell_operator_app.SetupLogging() + log.Infof("%s %s, shell-operator %s", app.AppName, app.Version, shell_operator_app.Version) + // Be a good parent - clean up after the child processes // in case if addon-operator is a PID 1 process. go executor.Reap() diff --git a/go.mod b/go.mod index 0e1f2269..0ef040ab 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.12 require ( github.com/evanphx/json-patch v4.5.0+incompatible - github.com/flant/shell-operator v1.0.0-beta.5.0.20191023115920-4f35920cc42f // branch: json_logging + github.com/flant/shell-operator v1.0.0-beta.5.0.20191023161047-91e8e7a8683e // branch: json_logging github.com/go-openapi/spec v0.19.3 github.com/kennygrant/sanitize v1.2.4 github.com/otiai10/copy v1.0.1 diff --git a/go.sum b/go.sum index fca7ddd8..aaaadf03 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,7 @@ github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71 h1:q github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71/go.mod h1:KQ6PubnYxbkW8BoAXfWVra5dGuMSrec/G4ztFQEC2jU= github.com/flant/shell-operator v1.0.0-beta.5.0.20191023115920-4f35920cc42f h1:UwikOFWT3eCWIRYNugKHdZPa19mhIMEd1VWtNXoNpos= github.com/flant/shell-operator v1.0.0-beta.5.0.20191023115920-4f35920cc42f/go.mod h1:KQ6PubnYxbkW8BoAXfWVra5dGuMSrec/G4ztFQEC2jU= +github.com/flant/shell-operator v1.0.0-beta.5.0.20191023161047-91e8e7a8683e/go.mod h1:KQ6PubnYxbkW8BoAXfWVra5dGuMSrec/G4ztFQEC2jU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= diff --git a/pkg/addon-operator/start.go b/pkg/addon-operator/start.go index 4ef21cc7..95997f91 100644 --- a/pkg/addon-operator/start.go +++ b/pkg/addon-operator/start.go @@ -5,8 +5,6 @@ import ( log "github.com/sirupsen/logrus" - shell_operator_app "github.com/flant/shell-operator/pkg/app" - "github.com/flant/addon-operator/pkg/app" ) @@ -20,8 +18,6 @@ func Start() { os.Exit(1) } - log.Infof("addon-operator %s, shell-operator %s", app.Version, shell_operator_app.Version) - err = Init() if err != nil { log.Errorf("INIT failed: %v", err) From 4b802016fda9980938cebe609842f5f1472b22e6 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Tue, 29 Oct 2019 14:44:16 +0300 Subject: [PATCH 6/8] feat: json logging - ignore Synchronization event for v0 config --- .../hook/kube_event/hooks_controller.go | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/pkg/module_manager/hook/kube_event/hooks_controller.go b/pkg/module_manager/hook/kube_event/hooks_controller.go index 202d9e24..a68a5050 100644 --- a/pkg/module_manager/hook/kube_event/hooks_controller.go +++ b/pkg/module_manager/hook/kube_event/hooks_controller.go @@ -169,30 +169,40 @@ func (c *kubernetesHooksController) DisableModuleHooks(moduleName string, logLab func (c *kubernetesHooksController) HandleEvent(kubeEvent kube_events_manager.KubeEvent, logLabels map[string]string) ([]task.Task, error) { res := make([]task.Task, 0) - globalHook, hasGlobalHook := c.GlobalHooks[kubeEvent.ConfigId] - moduleHook, hasModuleHook := c.ModuleHooks[kubeEvent.ConfigId] + globalEventHook, hasGlobalHook := c.GlobalHooks[kubeEvent.ConfigId] + moduleEventHook, hasModuleHook := c.ModuleHooks[kubeEvent.ConfigId] if !hasGlobalHook && !hasModuleHook { return nil, fmt.Errorf("Possible a bug: kubernetes event '%s/%s/%s %s' is received, but no hook is found", kubeEvent.Namespace, kubeEvent.Kind, kubeEvent.Name, kubeEvent.Event) } + globalHook, _ := c.moduleManager.GetGlobalHook(globalEventHook.HookName) + moduleHook, _ := c.moduleManager.GetModuleHook(moduleEventHook.HookName) + hookLabels := utils.MergeLabels(logLabels) var taskType task.TaskType var kubeHook *kube_event.KubeEventHook + var configVersion string if hasGlobalHook { - kubeHook = globalHook taskType = task.GlobalHookRun - hookLabels["hook"] = globalHook.HookName + kubeHook = globalEventHook + configVersion = globalHook.Config.Version + hookLabels["hook"] = globalEventHook.HookName hookLabels["hook.type"] = "global" } else { - kubeHook = moduleHook taskType = task.ModuleHookRun - hookLabels["hook"] = moduleHook.HookName + kubeHook = moduleEventHook + configVersion = moduleHook.Config.Version + hookLabels["hook"] = moduleEventHook.HookName hookLabels["hook.type"] = "module" } switch kubeEvent.Type { case "Synchronization": + // Ignore Synchronization for v0 + if configVersion == "v0" { + break + } // Send all objects objList := make([]interface{}, 0) for _, obj := range kubeEvent.Objects { From 03576e75cedd0562e53848731ca2ecf1da375543 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Tue, 29 Oct 2019 16:30:45 +0300 Subject: [PATCH 7/8] fix: empty binding context on v0 config --- pkg/module_manager/hook/kube_event/hooks_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/module_manager/hook/kube_event/hooks_controller.go b/pkg/module_manager/hook/kube_event/hooks_controller.go index a68a5050..08b22a4a 100644 --- a/pkg/module_manager/hook/kube_event/hooks_controller.go +++ b/pkg/module_manager/hook/kube_event/hooks_controller.go @@ -175,8 +175,6 @@ func (c *kubernetesHooksController) HandleEvent(kubeEvent kube_events_manager.Ku return nil, fmt.Errorf("Possible a bug: kubernetes event '%s/%s/%s %s' is received, but no hook is found", kubeEvent.Namespace, kubeEvent.Kind, kubeEvent.Name, kubeEvent.Event) } - globalHook, _ := c.moduleManager.GetGlobalHook(globalEventHook.HookName) - moduleHook, _ := c.moduleManager.GetModuleHook(moduleEventHook.HookName) hookLabels := utils.MergeLabels(logLabels) @@ -186,12 +184,14 @@ func (c *kubernetesHooksController) HandleEvent(kubeEvent kube_events_manager.Ku if hasGlobalHook { taskType = task.GlobalHookRun kubeHook = globalEventHook + globalHook, _ := c.moduleManager.GetGlobalHook(globalEventHook.HookName) configVersion = globalHook.Config.Version hookLabels["hook"] = globalEventHook.HookName hookLabels["hook.type"] = "global" } else { taskType = task.ModuleHookRun kubeHook = moduleEventHook + moduleHook, _ := c.moduleManager.GetModuleHook(moduleEventHook.HookName) configVersion = moduleHook.Config.Version hookLabels["hook"] = moduleEventHook.HookName hookLabels["hook.type"] = "module" From 1cf7854d3c18f7ae1e4e36415770eb8291259673 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Tue, 24 Sep 2019 20:29:14 +0300 Subject: [PATCH 8/8] feat: use werf to install modules' charts --- Dockerfile-alpine3.9 | 3 +- go.sum | 1 + pkg/app/app.go | 3 ++ pkg/helm/helm.go | 6 ++- pkg/helm/werf.go | 101 +++++++++++++++++++++++++++++++++++ pkg/module_manager/module.go | 4 +- 6 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 pkg/helm/werf.go diff --git a/Dockerfile-alpine3.9 b/Dockerfile-alpine3.9 index 9857b283..a99d12cd 100644 --- a/Dockerfile-alpine3.9 +++ b/Dockerfile-alpine3.9 @@ -13,7 +13,8 @@ RUN apk --no-cache add ca-certificates jq bash && \ tar -z -x -C /bin -f /helm.tgz --strip-components=1 linux-amd64/helm linux-amd64/tiller && \ rm -f /helm.tgz && \ helm init --client-only && \ - mkdir /hooks + wget https://dl.bintray.com/flant/werf/v1.0.4-beta.12/werf-linux-amd64-v1.0.4-beta.12 -O /bin/werf && \ + chmod +x /bin/werf COPY --from=0 /addon-operator/addon-operator / WORKDIR / ENV MODULES_DIR /modules diff --git a/go.sum b/go.sum index aaaadf03..db17d527 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,7 @@ github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71 h1:q github.com/flant/shell-operator v1.0.0-beta.5.0.20191014100941-a02a093d4a71/go.mod h1:KQ6PubnYxbkW8BoAXfWVra5dGuMSrec/G4ztFQEC2jU= github.com/flant/shell-operator v1.0.0-beta.5.0.20191023115920-4f35920cc42f h1:UwikOFWT3eCWIRYNugKHdZPa19mhIMEd1VWtNXoNpos= github.com/flant/shell-operator v1.0.0-beta.5.0.20191023115920-4f35920cc42f/go.mod h1:KQ6PubnYxbkW8BoAXfWVra5dGuMSrec/G4ztFQEC2jU= +github.com/flant/shell-operator v1.0.0-beta.5.0.20191023161047-91e8e7a8683e h1:TndP4g22Ww+AZxsipjg6qIl/unZReraHa68dLX08bZo= github.com/flant/shell-operator v1.0.0-beta.5.0.20191023161047-91e8e7a8683e/go.mod h1:KQ6PubnYxbkW8BoAXfWVra5dGuMSrec/G4ztFQEC2jU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= diff --git a/pkg/app/app.go b/pkg/app/app.go index 956c1918..7a8f0ada 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -23,6 +23,9 @@ var TillerProbeListenAddress = "127.0.0.1" var TillerProbeListenPort int32 = 44435 var TillerMaxHistory = 0 +var WerfTillerNamespace = "" +var WerfArgs = "" + var ConfigMapName = "addon-operator" var ValuesChecksumsAnnotation = "addon-operator/values-checksums" var TasksQueueDumpFilePath = "/tmp/addon-operator-tasks-queue" diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 66a4f77a..696b8387 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -14,10 +14,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kblabels "k8s.io/apimachinery/pkg/labels" - "github.com/flant/addon-operator/pkg/app" - "github.com/flant/addon-operator/pkg/utils" "github.com/flant/shell-operator/pkg/executor" "github.com/flant/shell-operator/pkg/kube" + + "github.com/flant/addon-operator/pkg/app" + "github.com/flant/addon-operator/pkg/utils" + ) const HelmPath = "helm" diff --git a/pkg/helm/werf.go b/pkg/helm/werf.go new file mode 100644 index 00000000..c454dd82 --- /dev/null +++ b/pkg/helm/werf.go @@ -0,0 +1,101 @@ +package helm + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "strings" + + log "github.com/sirupsen/logrus" + + "github.com/flant/shell-operator/pkg/executor" + + "github.com/flant/addon-operator/pkg/app" +) + +const WerfPath = "werf" + +type WerfClient interface { + DeployChart(releaseName string, chart string, valuesPaths []string, setValues []string, namespace string) error +} + +// WerfOptions +// FIXME is this needed? +type WerfOptions struct { + HelmReleaseStorageNamespace string + Namespace string +} + +type werfClient struct { + Options WerfOptions + LogEntry *log.Entry +} + +// werfClient implements WerfClient +var _ WerfClient = &werfClient{} + +func NewWerfClient(logEntry *log.Entry, opts WerfOptions) WerfClient { + return &werfClient{ + LogEntry: logEntry.WithField("operator.component", "werf"), + Options: opts, + } +} + +func (w *werfClient) DeployChart(releaseName string, chart string, valuesPaths []string, setValues []string, namespace string) error { + args := make([]string, 0) + args = append(args, "helm") + args = append(args, "deploy-chart") + + ns := namespace + if app.WerfTillerNamespace != "" { + ns = app.WerfTillerNamespace + } + args = append(args, "--namespace") + args = append(args, ns) + args = append(args, "--helm-release-storage-namespace") + args = append(args, ns) + + for _, valuesPath := range valuesPaths { + args = append(args, "--values") + args = append(args, valuesPath) + } + + for _, setValue := range setValues { + args = append(args, "--set") + args = append(args, setValue) + } + + args = append(args, chart) + args = append(args, releaseName) + + w.LogEntry.Infof("Running werf helm deploy-chart for release '%s' with chart '%s' in namespace '%s' ...", releaseName, chart, ns) + stdout, stderr, err := w.Run(args...) + if err != nil { + return fmt.Errorf("werf helm deploy-chart failed: %s:\n%s %s", err, stdout, stderr) + } + w.LogEntry.Infof("werf helm deploy-chart for release '%s' with chart '%s' in namespace '%s' was successful:\n%s\n%s", releaseName, chart, ns, stdout, stderr) + + return nil +} + +// Cmd starts Helm with specified arguments. +// Sets the TILLER_NAMESPACE environment variable before starting, because Addon-operator works with its own Tiller. +func (w *werfClient) Run(args ...string) (stdout string, stderr string, err error) { + cmd := exec.Command(WerfPath, args...) + cmd.Env = os.Environ() + if app.WerfTillerNamespace != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("TILLER_NAMESPACE=%s", app.WerfTillerNamespace)) + } + + var stdoutBuf bytes.Buffer + cmd.Stdout = &stdoutBuf + var stderrBuf bytes.Buffer + cmd.Stderr = &stderrBuf + + err = executor.Run(cmd) + stdout = strings.TrimSpace(stdoutBuf.String()) + stderr = strings.TrimSpace(stderrBuf.String()) + + return +} diff --git a/pkg/module_manager/module.go b/pkg/module_manager/module.go index c21b00e1..b084578d 100644 --- a/pkg/module_manager/module.go +++ b/pkg/module_manager/module.go @@ -215,11 +215,11 @@ func (m *Module) runHelmInstall() error { if doRelease { logEntry.Debugf("helm release '%s' checksum '%s': installing/upgrading release", helmReleaseName, checksum) - return helmClient.UpgradeRelease( + werfCl := helm.NewWerfClient(logEntry, helm.WerfOptions{}) + return werfCl.DeployChart( helmReleaseName, runChartPath, []string{valuesPath}, []string{fmt.Sprintf("_addonOperatorModuleChecksum=%s", checksum)}, - //helm.Client.TillerNamespace(), app.Namespace, ) } else {