From f32e1667f02d5853bf1b02d1d5802e5592671366 Mon Sep 17 00:00:00 2001 From: Kim Tao Date: Wed, 29 Jan 2025 15:20:46 -0500 Subject: [PATCH 1/9] Add host env vars for user data --- globals.go | 3 +++ model/host/hostutil.go | 39 +++++++++++++++++++++++++++++++++++---- operations/agent.go | 20 ++++++++++++++++---- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/globals.go b/globals.go index 86cedccb562..49deba7919d 100644 --- a/globals.go +++ b/globals.go @@ -262,6 +262,9 @@ const ( AgentMonitorTag = "agent-monitor" HostFetchTag = "host-fetch" + HostIDEnvVar = "HOST_ID" + HostSecretEnvVar = "HOST_SECRET" + DegradedLoggingPercent = 10 SetupScriptName = "setup.sh" diff --git a/model/host/hostutil.go b/model/host/hostutil.go index 959c1dfdd60..bb84efbca93 100644 --- a/model/host/hostutil.go +++ b/model/host/hostutil.go @@ -1019,8 +1019,11 @@ func (h *Host) AgentCommand(settings *evergreen.Settings, executablePath string) "agent", fmt.Sprintf("--api_server=%s", settings.Api.URL), "--mode=host", - fmt.Sprintf("--host_id=%s", h.Id), - fmt.Sprintf("--host_secret=%s", h.Secret), + // kim: TODO: this needs to stay for the deploy, but commenting it out + // verifies if the env vars get propagated to the agent properly via the + // agent monitor. + // fmt.Sprintf("--host_id=%s", h.Id), + // fmt.Sprintf("--host_secret=%s", h.Secret), fmt.Sprintf("--provider=%s", h.Distro.Provider), "--log_output=file", fmt.Sprintf("--log_prefix=%s", filepath.Join(h.Distro.WorkDir, "agent")), @@ -1029,6 +1032,28 @@ func (h *Host) AgentCommand(settings *evergreen.Settings, executablePath string) } } +// kim: TODO: need to test the agent with env vars for legacy SSH, SSH, and user +// data provisioning. Also need to check reprovisioning works. + +// AgentEnv returns the environment variables required to start the agent. +func (h *Host) AgentEnv() map[string]string { + return map[string]string{ + evergreen.HostIDEnvVar: h.Id, + evergreen.HostSecretEnvVar: h.Secret, + } +} + +// AgentEnvSlice is the same as AgentEnv but returns the environment variables +// as a slice of key=value strings. +func (h *Host) AgentEnvSlice() []string { + env := h.AgentEnv() + var envSlice []string + for k, v := range env { + envSlice = append(envSlice, fmt.Sprintf("%s=%s", k, v)) + } + return envSlice +} + // AgentMonitorOptions assembles the input to a Jasper request to start the // agent monitor. func (h *Host) AgentMonitorOptions(settings *evergreen.Settings) *options.Create { @@ -1036,6 +1061,11 @@ func (h *Host) AgentMonitorOptions(settings *evergreen.Settings) *options.Create credsPath := h.Distro.AbsPathNotCygwinCompatible(h.Distro.BootstrapSettings.JasperCredentialsPath) shellPath := h.Distro.AbsPathNotCygwinCompatible(h.Distro.BootstrapSettings.ShellPath) + // kim: NOTE: need to pass along host ID and host secret env vars from the + // agent monitor to the agent. Potentially, it will just work with no + // changes to the agent monitor if the agent inherits the env vars from the + // agent monitor, but would have to check. If not, will have to explicitly + // set it when invoking the agent. args := append(h.AgentCommand(settings, ""), "monitor") args = append(args, fmt.Sprintf("--client_path=%s", clientPath), @@ -1048,8 +1078,9 @@ func (h *Host) AgentMonitorOptions(settings *evergreen.Settings) *options.Create ) return &options.Create{ - Args: args, - Tags: []string{evergreen.AgentMonitorTag}, + Args: args, + Environment: h.AgentEnv(), + Tags: []string{evergreen.AgentMonitorTag}, } } diff --git a/operations/agent.go b/operations/agent.go index b238e5b6163..c7ef85773ee 100644 --- a/operations/agent.go +++ b/operations/agent.go @@ -49,12 +49,14 @@ func Agent() cli.Command { }, Flags: []cli.Flag{ cli.StringFlag{ - Name: agentHostIDFlagName, - Usage: "the ID of the host the agent is running on (applies only to host mode)", + Name: agentHostIDFlagName, + Usage: "the ID of the host the agent is running on (applies only to host mode)", + EnvVar: evergreen.HostIDEnvVar, }, cli.StringFlag{ - Name: agentHostSecretFlagName, - Usage: "secret for the current host (applies only to host mode)", + Name: agentHostSecretFlagName, + Usage: "secret for the current host (applies only to host mode)", + EnvVar: evergreen.HostSecretEnvVar, }, cli.StringFlag{ Name: podIDFlagName, @@ -159,6 +161,16 @@ func Agent() cli.Command { SendTaskLogsToGlobalSender: c.Bool(sendTaskLogsToGlobalSenderFlagName), } + // Once the agent has retrieved the host ID and secret, unset those + // env vars to prevent them from leaking to task processes such as + // shell.exec and subprocess.exec. + if err := os.Unsetenv(evergreen.HostIDEnvVar); err != nil { + return errors.Wrapf(err, "unsetting %s env var", evergreen.HostIDEnvVar) + } + if err := os.Unsetenv(evergreen.HostSecretEnvVar); err != nil { + return errors.Wrapf(err, "unsetting %s env var", evergreen.HostSecretEnvVar) + } + if err := os.MkdirAll(opts.WorkingDirectory, 0777); err != nil { return errors.Wrapf(err, "creating working directory '%s'", opts.WorkingDirectory) } From 38f9b8fc6e154f400037f560f7fc7ace2a878dba Mon Sep 17 00:00:00 2001 From: Kim Tao Date: Wed, 29 Jan 2025 15:22:24 -0500 Subject: [PATCH 2/9] Add host env vars for legacy SSH provisioning --- units/provisioning_agent_deploy.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/units/provisioning_agent_deploy.go b/units/provisioning_agent_deploy.go index 025641d9cda..b7f5cd6b57c 100644 --- a/units/provisioning_agent_deploy.go +++ b/units/provisioning_agent_deploy.go @@ -292,6 +292,7 @@ func (j *agentDeployJob) prepRemoteHost(ctx context.Context) error { // Start the agent process on the specified remote host. func (j *agentDeployJob) startAgentOnRemote(ctx context.Context, settings *evergreen.Settings) (string, error) { // build the command to run on the remote machine + // kim: NOTE: need to modify this to set the env vars. remoteCmd := strings.Join(j.host.AgentCommand(settings, ""), " ") grip.Info(message.Fields{ "message": "starting agent on host", @@ -302,7 +303,9 @@ func (j *agentDeployJob) startAgentOnRemote(ctx context.Context, settings *everg ctx, cancel := context.WithTimeout(ctx, startAgentTimeout) defer cancel() - startAgentCmd := fmt.Sprintf("nohup %s > /tmp/start 2>&1 &", remoteCmd) + agentEnvVars := strings.Join(j.host.AgentEnvSlice(), " ") + // kim: TODO: test if setting these env vars really works with nohup. + startAgentCmd := fmt.Sprintf("%s nohup %s > /tmp/start 2>&1 &", agentEnvVars, remoteCmd) logs, err := j.host.RunSSHCommand(ctx, startAgentCmd) if err != nil { return logs, errors.Wrapf(err, "starting agent on host '%s'", j.host.Id) From 22e292041d0a54eacbb920cec4b14d784ed0dcb5 Mon Sep 17 00:00:00 2001 From: Kim Tao Date: Wed, 29 Jan 2025 15:22:45 -0500 Subject: [PATCH 3/9] Add container env vars for Docker container pools --- cloud/docker_client.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloud/docker_client.go b/cloud/docker_client.go index 540149382c1..ed2596b21c2 100644 --- a/cloud/docker_client.go +++ b/cloud/docker_client.go @@ -366,6 +366,8 @@ func (c *dockerClientImpl) CreateContainer(ctx context.Context, parentHost, cont // Build Evergreen agent command. agentCmdParts = containerHost.AgentCommand(c.evergreenSettings, pathToExecutable) containerHost.DockerOptions.Command = strings.Join(agentCmdParts, "\n") + // kim: TODO: need to test that env vars work for Docker containers. + containerHost.DockerOptions.EnvironmentVars = append(containerHost.DockerOptions.EnvironmentVars, containerHost.AgentEnvSlice()...) } // Populate container settings with command and new image. From 07b9953855a209e092fd76b4a30f663edac5eae0 Mon Sep 17 00:00:00 2001 From: Kim Tao Date: Wed, 29 Jan 2025 15:23:11 -0500 Subject: [PATCH 4/9] Update docs to correct expansion redaction --- agent/globals/globals.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/globals/globals.go b/agent/globals/globals.go index 2f151bed334..114c0d46177 100644 --- a/agent/globals/globals.go +++ b/agent/globals/globals.go @@ -114,7 +114,7 @@ const ( AWSSessionToken = "AWS_SESSION_TOKEN" // AWSRoleExpiration is the expansion name for the expiration of a temporary AWS access key. AWSRoleExpiration = "AWS_ROLE_EXPIRATION" - // HostSecret is the environment variable within the agent that is unique to its running host. + // HostSecret is the expansion name within the agent that is the host's unique secret. HostSecret = "HOST_SECRET" ) From 1b046134d8bba069a42847ac1c8da7c7987cd13a Mon Sep 17 00:00:00 2001 From: Kim Tao Date: Wed, 29 Jan 2025 15:23:21 -0500 Subject: [PATCH 5/9] Update agent version --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index c73ea260a38..b5d96165bc5 100644 --- a/config.go +++ b/config.go @@ -37,7 +37,7 @@ var ( // Agent version to control agent rollover. The format is the calendar date // (YYYY-MM-DD). - AgentVersion = "2025-01-23" + AgentVersion = "2025-01-29" ) const ( From 28ef67596383d0761384d986e476c042083e7693 Mon Sep 17 00:00:00 2001 From: Kim Tao Date: Wed, 29 Jan 2025 16:44:09 -0500 Subject: [PATCH 6/9] Update notes to self --- model/host/hostutil.go | 12 ++++-------- operations/agent.go | 4 ++-- units/provisioning_agent_deploy.go | 1 - 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/model/host/hostutil.go b/model/host/hostutil.go index bb84efbca93..f6eeda598bf 100644 --- a/model/host/hostutil.go +++ b/model/host/hostutil.go @@ -1019,9 +1019,9 @@ func (h *Host) AgentCommand(settings *evergreen.Settings, executablePath string) "agent", fmt.Sprintf("--api_server=%s", settings.Api.URL), "--mode=host", - // kim: TODO: this needs to stay for the deploy, but commenting it out - // verifies if the env vars get propagated to the agent properly via the - // agent monitor. + // kim: TODO: this needs to stay for the deploy to ensure a smooth + // rollover, but commenting it out verifies if the env vars get + // propagated to the agent properly via the agent monitor. // fmt.Sprintf("--host_id=%s", h.Id), // fmt.Sprintf("--host_secret=%s", h.Secret), fmt.Sprintf("--provider=%s", h.Distro.Provider), @@ -1061,11 +1061,7 @@ func (h *Host) AgentMonitorOptions(settings *evergreen.Settings) *options.Create credsPath := h.Distro.AbsPathNotCygwinCompatible(h.Distro.BootstrapSettings.JasperCredentialsPath) shellPath := h.Distro.AbsPathNotCygwinCompatible(h.Distro.BootstrapSettings.ShellPath) - // kim: NOTE: need to pass along host ID and host secret env vars from the - // agent monitor to the agent. Potentially, it will just work with no - // changes to the agent monitor if the agent inherits the env vars from the - // agent monitor, but would have to check. If not, will have to explicitly - // set it when invoking the agent. + // kim: NOTE: agent monitor already passes on its environment to the agent. args := append(h.AgentCommand(settings, ""), "monitor") args = append(args, fmt.Sprintf("--client_path=%s", clientPath), diff --git a/operations/agent.go b/operations/agent.go index c7ef85773ee..d7494cb31c0 100644 --- a/operations/agent.go +++ b/operations/agent.go @@ -162,8 +162,8 @@ func Agent() cli.Command { } // Once the agent has retrieved the host ID and secret, unset those - // env vars to prevent them from leaking to task processes such as - // shell.exec and subprocess.exec. + // env vars to prevent them from being inherited by subprocesses + // such as shell.exec and subprocess.exec. if err := os.Unsetenv(evergreen.HostIDEnvVar); err != nil { return errors.Wrapf(err, "unsetting %s env var", evergreen.HostIDEnvVar) } diff --git a/units/provisioning_agent_deploy.go b/units/provisioning_agent_deploy.go index b7f5cd6b57c..f2f5d153e4a 100644 --- a/units/provisioning_agent_deploy.go +++ b/units/provisioning_agent_deploy.go @@ -292,7 +292,6 @@ func (j *agentDeployJob) prepRemoteHost(ctx context.Context) error { // Start the agent process on the specified remote host. func (j *agentDeployJob) startAgentOnRemote(ctx context.Context, settings *evergreen.Settings) (string, error) { // build the command to run on the remote machine - // kim: NOTE: need to modify this to set the env vars. remoteCmd := strings.Join(j.host.AgentCommand(settings, ""), " ") grip.Info(message.Fields{ "message": "starting agent on host", From 6f0e7399480da479084920fb5846ef628ef4796f Mon Sep 17 00:00:00 2001 From: Kim Tao Date: Thu, 30 Jan 2025 14:38:05 -0500 Subject: [PATCH 7/9] Update comment a little --- operations/agent.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operations/agent.go b/operations/agent.go index d7494cb31c0..86523dce8b6 100644 --- a/operations/agent.go +++ b/operations/agent.go @@ -162,8 +162,8 @@ func Agent() cli.Command { } // Once the agent has retrieved the host ID and secret, unset those - // env vars to prevent them from being inherited by subprocesses - // such as shell.exec and subprocess.exec. + // env vars to prevent them from being inherited by task + // subprocesses (e.g. shell.exec). if err := os.Unsetenv(evergreen.HostIDEnvVar); err != nil { return errors.Wrapf(err, "unsetting %s env var", evergreen.HostIDEnvVar) } From 3511a585554f03f0945ce9d32ee98f4ba81ee829 Mon Sep 17 00:00:00 2001 From: Kim Tao Date: Thu, 30 Jan 2025 16:38:27 -0500 Subject: [PATCH 8/9] Reintroduce host ID/secret flags for CLI temporarily while rolling over agent monitors on static hosts --- cloud/docker_client.go | 1 - model/host/hostutil.go | 14 +++++--------- units/provisioning_agent_deploy.go | 1 - 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/cloud/docker_client.go b/cloud/docker_client.go index ed2596b21c2..5e7f64f31a7 100644 --- a/cloud/docker_client.go +++ b/cloud/docker_client.go @@ -366,7 +366,6 @@ func (c *dockerClientImpl) CreateContainer(ctx context.Context, parentHost, cont // Build Evergreen agent command. agentCmdParts = containerHost.AgentCommand(c.evergreenSettings, pathToExecutable) containerHost.DockerOptions.Command = strings.Join(agentCmdParts, "\n") - // kim: TODO: need to test that env vars work for Docker containers. containerHost.DockerOptions.EnvironmentVars = append(containerHost.DockerOptions.EnvironmentVars, containerHost.AgentEnvSlice()...) } diff --git a/model/host/hostutil.go b/model/host/hostutil.go index f6eeda598bf..0af47c7a70e 100644 --- a/model/host/hostutil.go +++ b/model/host/hostutil.go @@ -1019,11 +1019,11 @@ func (h *Host) AgentCommand(settings *evergreen.Settings, executablePath string) "agent", fmt.Sprintf("--api_server=%s", settings.Api.URL), "--mode=host", - // kim: TODO: this needs to stay for the deploy to ensure a smooth - // rollover, but commenting it out verifies if the env vars get - // propagated to the agent properly via the agent monitor. - // fmt.Sprintf("--host_id=%s", h.Id), - // fmt.Sprintf("--host_secret=%s", h.Secret), + // TODO (DEVPROD-8409): delete the host ID and secret from the CLI args + // once the agent monitor and agent on all static/dynamic hosts have + // rolled over to newer versions. + fmt.Sprintf("--host_id=%s", h.Id), + fmt.Sprintf("--host_secret=%s", h.Secret), fmt.Sprintf("--provider=%s", h.Distro.Provider), "--log_output=file", fmt.Sprintf("--log_prefix=%s", filepath.Join(h.Distro.WorkDir, "agent")), @@ -1032,9 +1032,6 @@ func (h *Host) AgentCommand(settings *evergreen.Settings, executablePath string) } } -// kim: TODO: need to test the agent with env vars for legacy SSH, SSH, and user -// data provisioning. Also need to check reprovisioning works. - // AgentEnv returns the environment variables required to start the agent. func (h *Host) AgentEnv() map[string]string { return map[string]string{ @@ -1061,7 +1058,6 @@ func (h *Host) AgentMonitorOptions(settings *evergreen.Settings) *options.Create credsPath := h.Distro.AbsPathNotCygwinCompatible(h.Distro.BootstrapSettings.JasperCredentialsPath) shellPath := h.Distro.AbsPathNotCygwinCompatible(h.Distro.BootstrapSettings.ShellPath) - // kim: NOTE: agent monitor already passes on its environment to the agent. args := append(h.AgentCommand(settings, ""), "monitor") args = append(args, fmt.Sprintf("--client_path=%s", clientPath), diff --git a/units/provisioning_agent_deploy.go b/units/provisioning_agent_deploy.go index f2f5d153e4a..d18ffb6c977 100644 --- a/units/provisioning_agent_deploy.go +++ b/units/provisioning_agent_deploy.go @@ -303,7 +303,6 @@ func (j *agentDeployJob) startAgentOnRemote(ctx context.Context, settings *everg defer cancel() agentEnvVars := strings.Join(j.host.AgentEnvSlice(), " ") - // kim: TODO: test if setting these env vars really works with nohup. startAgentCmd := fmt.Sprintf("%s nohup %s > /tmp/start 2>&1 &", agentEnvVars, remoteCmd) logs, err := j.host.RunSSHCommand(ctx, startAgentCmd) if err != nil { From 1c10ce4e332dc83865e63acb13ff4188ce794e60 Mon Sep 17 00:00:00 2001 From: Kim Tao Date: Thu, 30 Jan 2025 17:05:03 -0500 Subject: [PATCH 9/9] Update smoke test to use env vars for auth --- operations/service_deploy_smoke.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/operations/service_deploy_smoke.go b/operations/service_deploy_smoke.go index 9323069728e..61ba86ba24f 100644 --- a/operations/service_deploy_smoke.go +++ b/operations/service_deploy_smoke.go @@ -41,7 +41,7 @@ func startLocalEvergreen() cli.Command { return errors.Wrap(err, "getting working directory") } binary := filepath.Join(wd, "clients", runtime.GOOS+"_"+runtime.GOARCH, "evergreen") - if err := smokeRunBinary(exit, "web.service", wd, binary, "service", "web", "--db", "evergreen_local", "--testing-env"); err != nil { + if err := smokeRunBinary(exit, "web.service", wd, nil, binary, "service", "web", "--db", "evergreen_local", "--testing-env"); err != nil { return errors.Wrap(err, "running web service") } <-exit @@ -142,7 +142,7 @@ func smokeStartEvergreen() cli.Command { exit := make(chan error, 3) if startWeb { - if err := smokeRunBinary(exit, "web.service", wd, binary, "service", "web", "--testing-env", "--conf", confPath); err != nil { + if err := smokeRunBinary(exit, "web.service", wd, nil, binary, "service", "web", "--testing-env", "--conf", confPath); err != nil { return errors.Wrap(err, "running web service") } } @@ -153,13 +153,26 @@ func smokeStartEvergreen() cli.Command { return errors.Wrap(err, "starting mock Cedar service") } + var envVars []string + switch mode { + case string(globals.HostMode): + envVars = []string{ + fmt.Sprintf("HOST_ID=%s", execModeID), + fmt.Sprintf("HOST_SECRET=%s", execModeSecret), + } + case string(globals.PodMode): + envVars = []string{ + fmt.Sprintf("POD_ID=%s", execModeID), + fmt.Sprintf("POD_SECRET=%s", execModeSecret), + } + } + err := smokeRunBinary(exit, "agent", wd, + envVars, binary, "agent", fmt.Sprintf("--mode=%s", mode), - fmt.Sprintf("--%s_id=%s", mode, execModeID), - fmt.Sprintf("--%s_secret=%s", mode, execModeSecret), "--api_server", apiServerURL, "--log_output", string(globals.LogOutputFile), "--log_prefix", "smoke.agent", @@ -208,11 +221,13 @@ func smokeStartEvergreen() cli.Command { exit, "agent.monitor", wd, + []string{ + fmt.Sprintf("HOST_ID=%s", execModeID), + fmt.Sprintf("HOST_SECRET=%s", execModeSecret), + }, binary, "agent", fmt.Sprintf("--mode=%s", globals.HostMode), - "--host_id", execModeID, - "--host_secret", execModeSecret, "--api_server", apiServerURL, "--log_output", string(globals.LogOutputFile), "--global_task_logs", @@ -239,9 +254,10 @@ func smokeStartEvergreen() cli.Command { } } -func smokeRunBinary(exit chan error, name, wd, bin string, cmdParts ...string) error { +func smokeRunBinary(exit chan error, name string, wd string, envVars []string, bin string, cmdParts ...string) error { cmd := exec.Command(bin, cmdParts...) cmd.Env = append(os.Environ(), fmt.Sprintf("EVGHOME=%s", wd)) + cmd.Env = append(cmd.Env, envVars...) cmdSender := send.NewWriterSender(send.MakeNative()) cmdSender.SetName(name) cmd.Stdout = cmdSender