From 2e3aea0f13b10f3d220dfa9561ba8e1c789fa361 Mon Sep 17 00:00:00 2001 From: Muthu Chidambaram Date: Thu, 10 Oct 2024 13:32:18 -0500 Subject: [PATCH] [BACKPORT 2.20.7][PLAT-11305] Adding prometheus log file Summary: Original commit: 2d77f2313506bce0d6028608bfc38c60295f81b6 / D38855 We were not passing the prometheus log file anywhere to actually write the logs but were showing it in the status output which was kind of confusing. This diff adds the StandardOutput and StandardError redirection fields directly to the service file. NOTE: This only works with systemd >= 236 which should cover most modern linux distros. If the version is < 240 we will use file mode which will replace the file on each restart of the service, but at least it's logged to a file in this case. Test Plan: install yba and make sure prometheus logs present Reviewers: dshubin, sanketh Reviewed By: dshubin Subscribers: yugaware, svc_phabricator, steve.varnau Differential Revision: https://phorge.dev.yugabyte.com/D38943 --- managed/yba-installer/cmd/prometheus.go | 19 +++++++++++- .../templates/yba-installer-prometheus.yml | 5 ++++ managed/yba-installer/pkg/common/utils.go | 17 +++++++++++ .../pkg/config/templatedConfig.go | 5 ++-- .../pkg/systemd/service_management.go | 29 +++++++++++++++++++ 5 files changed, 72 insertions(+), 3 deletions(-) diff --git a/managed/yba-installer/cmd/prometheus.go b/managed/yba-installer/cmd/prometheus.go index e0fdff58045f..448665a864bf 100644 --- a/managed/yba-installer/cmd/prometheus.go +++ b/managed/yba-installer/cmd/prometheus.go @@ -31,9 +31,14 @@ type prometheusDirectories struct { DataDir string PromDir string cronScript string + LogDir string } func newPrometheusDirectories() prometheusDirectories { + logDir := "/var/log" + if !common.HasSudoAccess() { + logDir = common.GetBaseInstall() + "/data/logs" + } return prometheusDirectories{ SystemdFileLocation: common.SystemdDir + "/prometheus.service", ConfFileLocation: common.GetSoftwareRoot() + "/prometheus/conf/prometheus.yml", @@ -43,6 +48,7 @@ func newPrometheusDirectories() prometheusDirectories { PromDir: common.GetSoftwareRoot() + "/prometheus", cronScript: filepath.Join( common.GetInstallerSoftwareDir(), common.CronDir, "managePrometheus.sh"), + LogDir: logDir, } } @@ -298,12 +304,17 @@ func (prom Prometheus) Upgrade() error { // Status prints out the header information for the // Prometheus service specifically. func (prom Prometheus) Status() (common.Status, error) { + + logFileLoc := common.GetBaseInstall() + "/data/logs/prometheus.log" + if common.SystemdLogMethod() == "" { + logFileLoc = "journalctl -u prometheus" + } status := common.Status{ Service: prom.Name(), Port: viper.GetInt("prometheus.port"), Version: prom.version, ConfigLoc: prom.ConfFileLocation, - LogFileLoc: prom.DataDir + "/prometheus.log", + LogFileLoc: logFileLoc, } // Set the systemd service file location if one exists @@ -525,6 +536,12 @@ func (prom Prometheus) createPrometheusSymlinks() error { {promPkg, prom.PromDir, "consoles"}, {promPkg, prom.PromDir, "console_libraries"}, } + // for root the log file is in /var/log in case of SELinux + if (common.HasSudoAccess()) { + links = append(links, struct { + pkgDir, linkDir, binary string + }{prom.LogDir, filepath.Join(common.GetBaseInstall(), "data/logs"), "prometheus.log"}) + } for _, link := range links { if err := common.CreateSymlink(link.pkgDir, link.linkDir, link.binary); err != nil { log.Error("failed to create symlink for " + link.binary + ": " + err.Error()) diff --git a/managed/yba-installer/config/templates/yba-installer-prometheus.yml b/managed/yba-installer/config/templates/yba-installer-prometheus.yml index ceb4d17d0a09..87a91c610618 100644 --- a/managed/yba-installer/config/templates/yba-installer-prometheus.yml +++ b/managed/yba-installer/config/templates/yba-installer-prometheus.yml @@ -200,6 +200,11 @@ Restart=always RestartSec={{ yamlPath "prometheus.restartSeconds"}} OOMScoreAdjust={{ yamlPath "prometheus.oomScoreAdjust"}} + {{ $systemdLog := systemdLogMethod }} + {{ if ne $systemdLog "" }} + StandardOutput={{ $systemdLog }}:{{ .LogDir }}/prometheus.log + StandardError={{ $systemdLog }}:{{ .LogDir }}/prometheus.log + {{ end }} [Install] WantedBy=multi-user.target diff --git a/managed/yba-installer/pkg/common/utils.go b/managed/yba-installer/pkg/common/utils.go index 1d966eeac2f6..8f1f39c8cfe7 100644 --- a/managed/yba-installer/pkg/common/utils.go +++ b/managed/yba-installer/pkg/common/utils.go @@ -27,6 +27,7 @@ import ( "github.com/yugabyte/yugabyte-db/managed/yba-installer/pkg/common/shell" log "github.com/yugabyte/yugabyte-db/managed/yba-installer/pkg/logging" + "github.com/yugabyte/yugabyte-db/managed/yba-installer/pkg/systemd" // "github.com/yugabyte/yugabyte-db/managed/yba-installer/preflight" ) @@ -124,6 +125,22 @@ func SplitInput(input string) []string { }) } +func SystemdLogMethod() string { + version, err := systemd.Version() + if err != nil { + log.Debug("Error determining systemd version: " + err.Error()) + return "" + } + log.Debug("Detected OS with systemd version " + strconv.Itoa(version)) + if version >= 240 { + return "append" + } + if version >= 236 { + return "file" + } + return "" +} + // Create or truncate a file at a relative path for the non-root case. Have to make the directory // before inserting the file in that directory. func Create(p string) (*os.File, error) { diff --git a/managed/yba-installer/pkg/config/templatedConfig.go b/managed/yba-installer/pkg/config/templatedConfig.go index 382187f3ca37..b338260a5eed 100644 --- a/managed/yba-installer/pkg/config/templatedConfig.go +++ b/managed/yba-installer/pkg/config/templatedConfig.go @@ -58,8 +58,9 @@ func readConfigAndTemplate(configYmlFileName string, service common.Component) ( "installRoot": common.GetSoftwareRoot, "installVersionDir": common.GetInstallerSoftwareDir, "baseInstall": common.GetBaseInstall, - "removeQuotes": common.RemoveQuotes, - "splitInput": common.SplitInput, + "splitInput": common.SplitInput, + "removeQuotes": common.RemoveQuotes, + "systemdLogMethod": common.SystemdLogMethod, } tmpl, err := template.New(configYmlFileName). diff --git a/managed/yba-installer/pkg/systemd/service_management.go b/managed/yba-installer/pkg/systemd/service_management.go index e265774b7c21..02859c3ef2de 100644 --- a/managed/yba-installer/pkg/systemd/service_management.go +++ b/managed/yba-installer/pkg/systemd/service_management.go @@ -2,6 +2,7 @@ package systemd import ( "fmt" + "strconv" "strings" "github.com/yugabyte/yugabyte-db/managed/yba-installer/pkg/common/shell" @@ -39,6 +40,34 @@ func DaemonReload() error { return nil } +// Version returns the systemd version on the system +func Version() (int, error) { + // Run the 'systemctl --version' command + output := shell.Run("systemctl", "--version") + if !output.SucceededOrLog() { + return 0, fmt.Errorf("unable to run systemctl --version") + } + + // Get the first line of the output, which contains the version number + lines := strings.Split(output.StdoutString(), "\n") + if len(lines) == 0 { + return 0, fmt.Errorf("unable to parse systemd version") + } + + // Extract the version number from the first line + fields := strings.Fields(lines[0]) + if len(fields) < 2 { + return 0, fmt.Errorf("unexpected output format") + } + + // Convert the version string to an integer + version, err := strconv.Atoi(fields[1]) + if err != nil { + return 0, fmt.Errorf("unable to parse version number: %v", err) + } + return version, nil +} + // runSysctlCmd is a helper for running some basic systemctl commands // cmd: systemctl cmd to run // services: list of services to perform the command against