[BACKPORT 2.20.7][PLAT-11305] Adding prometheus log file

Summary:
Original commit: 2d77f23 / D38855
We were not passing the prometheus log file anywhere to actually write the logs but were showing it in the status output which was kind of confusing. This diff adds the StandardOutput and StandardError redirection fields directly to the service file.

NOTE: This only works with systemd >= 236 which should cover most modern linux distros. If the version is < 240 we will use file mode which will replace the file on each restart of the service, but at least it's logged to a file in this case.

Test Plan: install yba and make sure prometheus logs present

Reviewers: dshubin, sanketh

Reviewed By: dshubin

Subscribers: yugaware, svc_phabricator, steve.varnau

Differential Revision: https://phorge.dev.yugabyte.com/D38943

managed/yba-installer/cmd/prometheus.go

@@ -31,9 +31,14 @@ type prometheusDirectories struct {
 	DataDir             string
 	PromDir             string
 	cronScript          string
+	LogDir							string
 }
 
 func newPrometheusDirectories() prometheusDirectories {
+	logDir := "/var/log"
+	if !common.HasSudoAccess() {
+		logDir = common.GetBaseInstall() + "/data/logs"
+	}
 	return prometheusDirectories{
 		SystemdFileLocation: common.SystemdDir + "/prometheus.service",
 		ConfFileLocation:    common.GetSoftwareRoot() + "/prometheus/conf/prometheus.yml",
@@ -43,6 +48,7 @@ func newPrometheusDirectories() prometheusDirectories {
 		PromDir:             common.GetSoftwareRoot() + "/prometheus",
 		cronScript: filepath.Join(
 			common.GetInstallerSoftwareDir(), common.CronDir, "managePrometheus.sh"),
+		LogDir:							 logDir,
 	}
 }
 
@@ -298,12 +304,17 @@ func (prom Prometheus) Upgrade() error {
 // Status prints out the header information for the
 // Prometheus service specifically.
 func (prom Prometheus) Status() (common.Status, error) {
+
+	logFileLoc := common.GetBaseInstall() + "/data/logs/prometheus.log"
+	if common.SystemdLogMethod() == "" {
+		logFileLoc = "journalctl -u prometheus"
+	}
 	status := common.Status{
 		Service:    prom.Name(),
 		Port:       viper.GetInt("prometheus.port"),
 		Version:    prom.version,
 		ConfigLoc:  prom.ConfFileLocation,
-		LogFileLoc: prom.DataDir + "/prometheus.log",
+		LogFileLoc: logFileLoc,
 	}
 
 	// Set the systemd service file location if one exists
@@ -525,6 +536,12 @@ func (prom Prometheus) createPrometheusSymlinks() error {
 		{promPkg, prom.PromDir, "consoles"},
 		{promPkg, prom.PromDir, "console_libraries"},
 	}
+	// for root the log file is in /var/log in case of SELinux
+	if (common.HasSudoAccess()) {
+		links = append(links, struct {
+			pkgDir, linkDir, binary string
+		}{prom.LogDir, filepath.Join(common.GetBaseInstall(), "data/logs"), "prometheus.log"})
+	}
 	for _, link := range links {
 		if err := common.CreateSymlink(link.pkgDir, link.linkDir, link.binary); err != nil {
 			log.Error("failed to create symlink for " + link.binary + ": " + err.Error())

managed/yba-installer/config/templates/yba-installer-prometheus.yml

@@ -200,6 +200,11 @@
       Restart=always
       RestartSec={{ yamlPath "prometheus.restartSeconds"}}
       OOMScoreAdjust={{ yamlPath "prometheus.oomScoreAdjust"}}
+      {{ $systemdLog := systemdLogMethod }}
+      {{ if ne $systemdLog "" }}
+      StandardOutput={{ $systemdLog }}:{{ .LogDir }}/prometheus.log
+      StandardError={{ $systemdLog }}:{{ .LogDir }}/prometheus.log
+      {{ end }}
 
       [Install]
       WantedBy=multi-user.target

managed/yba-installer/pkg/common/utils.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/yugabyte/yugabyte-db/managed/yba-installer/pkg/common/shell"
 	log "github.com/yugabyte/yugabyte-db/managed/yba-installer/pkg/logging"
+	"github.com/yugabyte/yugabyte-db/managed/yba-installer/pkg/systemd"
 	// "github.com/yugabyte/yugabyte-db/managed/yba-installer/preflight"
 )
 
@@ -124,6 +125,22 @@ func SplitInput(input string) []string {
 	})
 }
 
+func SystemdLogMethod() string {
+	version, err := systemd.Version()
+	if err != nil {
+		log.Debug("Error determining systemd version: " + err.Error())
+		return ""
+	}
+	log.Debug("Detected OS with systemd version " + strconv.Itoa(version))
+	if version >= 240 {
+		return "append"
+	}
+	if version >= 236 {
+		return "file"
+	}
+	return ""
+}
+
 // Create or truncate a file at a relative path for the non-root case. Have to make the directory
 // before inserting the file in that directory.
 func Create(p string) (*os.File, error) {

managed/yba-installer/pkg/config/templatedConfig.go

@@ -58,8 +58,9 @@ func readConfigAndTemplate(configYmlFileName string, service common.Component) (
 		"installRoot":       common.GetSoftwareRoot,
 		"installVersionDir": common.GetInstallerSoftwareDir,
 		"baseInstall":       common.GetBaseInstall,
-		"removeQuotes":		 	 common.RemoveQuotes,
-		"splitInput": 	 		 common.SplitInput,
+		"splitInput":        common.SplitInput,
+		"removeQuotes":      common.RemoveQuotes,
+		"systemdLogMethod":  common.SystemdLogMethod,
 	}
 
 	tmpl, err := template.New(configYmlFileName).

managed/yba-installer/pkg/systemd/service_management.go

@@ -2,6 +2,7 @@ package systemd
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/yugabyte/yugabyte-db/managed/yba-installer/pkg/common/shell"
@@ -39,6 +40,34 @@ func DaemonReload() error {
 	return nil
 }
 
+// Version returns the systemd version on the system
+func Version() (int, error) {
+		// Run the 'systemctl --version' command
+		output := shell.Run("systemctl", "--version")
+		if !output.SucceededOrLog() {
+			return 0, fmt.Errorf("unable to run systemctl --version")
+		}
+
+		// Get the first line of the output, which contains the version number
+		lines := strings.Split(output.StdoutString(), "\n")
+		if len(lines) == 0 {
+			return 0, fmt.Errorf("unable to parse systemd version")
+		}
+
+		// Extract the version number from the first line
+		fields := strings.Fields(lines[0])
+		if len(fields) < 2 {
+			return 0, fmt.Errorf("unexpected output format")
+		}
+
+		// Convert the version string to an integer
+		version, err := strconv.Atoi(fields[1])
+		if err != nil {
+			return 0, fmt.Errorf("unable to parse version number: %v", err)
+		}
+		return version, nil
+}
+
 // runSysctlCmd is a helper for running some basic systemctl commands
 // cmd: systemctl cmd to run
 // services: list of services to perform the command against