From 556bfc6e1574a6eb405b98441ce798267486b2c2 Mon Sep 17 00:00:00 2001 From: benpresnell <76139888+benpresnell@users.noreply.github.com> Date: Wed, 17 Nov 2021 09:22:00 -0500 Subject: [PATCH] http write timeout for the rest api frontend --- Jenkinsfile | 1 + cli/cmd/images.go | 46 ++++++-- cli/cmd/install.go | 93 +++++++++++++--- cli/k8s_client/types.go | 43 ++++++++ cli/k8s_client/yaml_factory.go | 100 +++++++++--------- cli/k8s_client/yaml_factory_test.go | 74 +++++++++++-- config/config.go | 3 +- frontend/rest/apiserver_http.go | 5 +- frontend/rest/apiserver_https.go | 5 +- .../templates/tridentorchestrator.yaml | 1 + helm/trident-operator/values.yaml | 4 + main.go | 29 ++++- .../orchestrator/apis/netapp/v1/types.go | 2 + .../orchestrator/installer/installer.go | 54 ++++++++-- 14 files changed, 360 insertions(+), 100 deletions(-) create mode 100644 cli/k8s_client/types.go diff --git a/Jenkinsfile b/Jenkinsfile index b66ece6ac..4146fa88b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,5 @@ @Library(['tools@master', 'tools-override@main']) _ + node { execute_pipeline(repository: 'trident') } diff --git a/cli/cmd/images.go b/cli/cmd/images.go index 3b2631325..7f528fe76 100644 --- a/cli/cmd/images.go +++ b/cli/cmd/images.go @@ -125,16 +125,46 @@ func getInstallYaml(semVersion *utils.Version) (string, error) { snapshotCRDVersion = client.GetSnapshotterCRDVersion() } - labels := make(map[string]string) - + deploymentArgs := &k8sclient.DeploymentYAMLArguments{ + DeploymentName: getDeploymentName(true), + TridentImage: tridentconfig.BuildImage, + AutosupportImage: tridentconfig.DefaultAutosupportImage, + AutosupportProxy: "", + AutosupportCustomURL: "", + AutosupportSerialNumber: "", + AutosupportHostname: "", + ImageRegistry: "", + LogFormat: "", + SnapshotCRDVersion: snapshotCRDVersion, + ImagePullSecrets: []string{}, + Labels: map[string]string{}, + ControllingCRDetails: map[string]string{}, + Debug: false, + UseIPv6: false, + SilenceAutosupport: true, + Version: semVersion, + TopologyEnabled: false, + HTTPRequestTimeout: tridentconfig.HTTPTimeoutString, + } // Get Deployment and Daemonset YAML and collect the names of the container images Trident needs to run. - yaml := k8sclient.GetCSIDeploymentYAML(getDeploymentName(true), - tridentconfig.BuildImage, tridentconfig.DefaultAutosupportImage, "", "", - "", "", "", "", snapshotCRDVersion, []string{}, nil, - nil, false, false, true, semVersion, false) + yaml := k8sclient.GetCSIDeploymentYAML(deploymentArgs) + daemonSetArgs := &k8sclient.DaemonsetYAMLArguments{ + DaemonsetName: "", + TridentImage: "", + ImageRegistry: "", + KubeletDir: "", + LogFormat: "", + ProbePort: "", + ImagePullSecrets: []string{}, + Labels: map[string]string{}, + ControllingCRDetails: map[string]string{}, + Debug: false, + NodePrep: false, + Version: semVersion, + HTTPRequestTimeout: tridentconfig.HTTPTimeoutString, + } // trident image here is an empty string because we are already going to get it from the deployment yaml - yaml += k8sclient.GetCSIDaemonSetYAML("", "", "", "", - "", "", []string{}, labels, nil, false, false, semVersion) + yaml += k8sclient.GetCSIDaemonSetYAML(daemonSetArgs) return yaml, nil } diff --git a/cli/cmd/install.go b/cli/cmd/install.go index ef08bab28..3f0ce1193 100644 --- a/cli/cmd/install.go +++ b/cli/cmd/install.go @@ -89,6 +89,7 @@ var ( logFormat string probePort int64 k8sTimeout time.Duration + httpRequestTimeout string // CLI-based K8S client client k8sclient.Interface @@ -154,6 +155,7 @@ func init() { installCmd.Flags().StringVar(&autosupportHostname, "autosupport-hostname", "", "The value to set for the hostname field in Autosupport payloads") installCmd.Flags().DurationVar(&k8sTimeout, "k8s-timeout", 180*time.Second, "The timeout for all Kubernetes operations.") + installCmd.Flags().StringVar(&httpRequestTimeout, "http-request-timeout", tridentconfig.HTTPTimeoutString, "The HTTP request timeout (e.g. '90s'.") if err := installCmd.Flags().MarkHidden("skip-k8s-version-check"); err != nil { _, _ = fmt.Fprintln(os.Stderr, err) @@ -437,17 +439,48 @@ func prepareYAMLFiles() error { return fmt.Errorf("could not write service YAML file; %v", err) } - deploymentYAML := k8sclient.GetCSIDeploymentYAML(getDeploymentName(true), - tridentImage, autosupportImage, autosupportProxy, autosupportCustomURL, autosupportSerialNumber, - autosupportHostname, imageRegistry, logFormat, snapshotCRDVersion, []string{}, labels, - nil, Debug, useIPv6, silenceAutosupport, client.ServerVersion(), topologyEnabled) + deploymentArgs := &k8sclient.DeploymentYAMLArguments{ + DeploymentName: getDeploymentName(true), + TridentImage: tridentImage, + AutosupportImage: autosupportImage, + AutosupportProxy: autosupportProxy, + AutosupportCustomURL: autosupportCustomURL, + AutosupportSerialNumber: autosupportSerialNumber, + AutosupportHostname: autosupportHostname, + ImageRegistry: imageRegistry, + LogFormat: logFormat, + SnapshotCRDVersion: snapshotCRDVersion, + ImagePullSecrets: []string{}, + Labels: labels, + ControllingCRDetails: nil, + Debug: Debug, + UseIPv6: useIPv6, + SilenceAutosupport: silenceAutosupport, + Version: client.ServerVersion(), + TopologyEnabled: topologyEnabled, + HTTPRequestTimeout: httpRequestTimeout, + } + deploymentYAML := k8sclient.GetCSIDeploymentYAML(deploymentArgs) if err = writeFile(deploymentPath, deploymentYAML); err != nil { return fmt.Errorf("could not write deployment YAML file; %v", err) } - daemonSetYAML := k8sclient.GetCSIDaemonSetYAML(getDaemonSetName(), - tridentImage, imageRegistry, kubeletDir, logFormat, strconv.FormatInt(probePort, 10), []string{}, - daemonSetlabels, nil, Debug, enableNodePrep, client.ServerVersion()) + daemonArgs := &k8sclient.DaemonsetYAMLArguments{ + DaemonsetName: getDaemonSetName(), + TridentImage: tridentImage, + ImageRegistry: imageRegistry, + KubeletDir: kubeletDir, + LogFormat: logFormat, + ProbePort: strconv.FormatInt(probePort, 10), + ImagePullSecrets: []string{}, + Labels: daemonSetlabels, + ControllingCRDetails: nil, + Debug: Debug, + NodePrep: enableNodePrep, + Version: client.ServerVersion(), + HTTPRequestTimeout: httpRequestTimeout, + } + daemonSetYAML := k8sclient.GetCSIDaemonSetYAML(daemonArgs) if err = writeFile(daemonsetPath, daemonSetYAML); err != nil { return fmt.Errorf("could not write daemonset YAML file; %v", err) } @@ -770,11 +803,29 @@ func installTrident() (returnError error) { returnError = client.CreateObjectByFile(deploymentPath) logFields = log.Fields{"path": deploymentPath} } else { + deploymentArgs := &k8sclient.DeploymentYAMLArguments{ + DeploymentName: getDeploymentName(true), + TridentImage: tridentImage, + AutosupportImage: autosupportImage, + AutosupportProxy: autosupportProxy, + AutosupportCustomURL: autosupportCustomURL, + AutosupportSerialNumber: autosupportSerialNumber, + AutosupportHostname: autosupportHostname, + ImageRegistry: imageRegistry, + LogFormat: logFormat, + SnapshotCRDVersion: snapshotCRDVersion, + ImagePullSecrets: []string{}, + Labels: labels, + ControllingCRDetails: nil, + Debug: Debug, + UseIPv6: useIPv6, + SilenceAutosupport: silenceAutosupport, + Version: client.ServerVersion(), + TopologyEnabled: topologyEnabled, + HTTPRequestTimeout: httpRequestTimeout, + } returnError = client.CreateObjectByYAML( - k8sclient.GetCSIDeploymentYAML(getDeploymentName(true), - tridentImage, autosupportImage, autosupportProxy, autosupportCustomURL, autosupportSerialNumber, - autosupportHostname, imageRegistry, logFormat, snapshotCRDVersion, []string{}, labels, nil, - Debug, useIPv6, silenceAutosupport, client.ServerVersion(), topologyEnabled)) + k8sclient.GetCSIDeploymentYAML(deploymentArgs)) logFields = log.Fields{} } if returnError != nil { @@ -795,11 +846,23 @@ func installTrident() (returnError error) { } else { daemonSetlabels := make(map[string]string) daemonSetlabels[appLabelKey] = TridentNodeLabelValue - + daemonSetArgs := &k8sclient.DaemonsetYAMLArguments{ + DaemonsetName: getDaemonSetName(), + TridentImage: tridentImage, + ImageRegistry: imageRegistry, + KubeletDir: kubeletDir, + LogFormat: logFormat, + ProbePort: strconv.FormatInt(probePort, 10), + ImagePullSecrets: []string{}, + Labels: daemonSetlabels, + ControllingCRDetails: nil, + Debug: Debug, + NodePrep: enableNodePrep, + Version: client.ServerVersion(), + HTTPRequestTimeout: httpRequestTimeout, + } returnError = client.CreateObjectByYAML( - k8sclient.GetCSIDaemonSetYAML(getDaemonSetName(), - tridentImage, imageRegistry, kubeletDir, logFormat, strconv.FormatInt(probePort, 10), []string{}, - daemonSetlabels, nil, Debug, enableNodePrep, client.ServerVersion())) + k8sclient.GetCSIDaemonSetYAML(daemonSetArgs)) logFields = log.Fields{} } if returnError != nil { diff --git a/cli/k8s_client/types.go b/cli/k8s_client/types.go new file mode 100644 index 000000000..fcdf03d13 --- /dev/null +++ b/cli/k8s_client/types.go @@ -0,0 +1,43 @@ +package k8sclient + +import ( + "github.com/netapp/trident/utils" +) + +type DeploymentYAMLArguments struct { + DeploymentName string `json:"deploymentName"` + TridentImage string `json:"tridentImage"` + AutosupportImage string `json:"autosupportImage"` + AutosupportProxy string `json:"autosupportProxy"` + AutosupportCustomURL string `json:"autosupportCustomURL"` + AutosupportSerialNumber string `json:"autosupportSerialNumber"` + AutosupportHostname string `json:"autosupportHostname"` + ImageRegistry string `json:"imageRegistry"` + LogFormat string `json:"logFormat"` + SnapshotCRDVersion string `json:"snapshotCRDVersion"` + ImagePullSecrets []string `json:"imagePullSecrets"` + Labels map[string]string `json:"labels"` + ControllingCRDetails map[string]string `json:"controllingCRDetails"` + Debug bool `json:"debug"` + UseIPv6 bool `json:"useIPv6"` + SilenceAutosupport bool `json:"silenceAutosupport"` + Version *utils.Version `json:"version"` + TopologyEnabled bool `json:"topologyEnabled"` + HTTPRequestTimeout string `json:"httpRequestTimeout"` +} + +type DaemonsetYAMLArguments struct { + DaemonsetName string `json:"daemonsetName"` + TridentImage string `json:"tridentImage"` + ImageRegistry string `json:"imageRegistry"` + KubeletDir string `json:"kubeletDir"` + LogFormat string `json:"logFormat"` + ProbePort string `json:"probePort"` + ImagePullSecrets []string `json:"imagePullSecrets"` + Labels map[string]string `json:"labels"` + ControllingCRDetails map[string]string `json:"controllingCRDetails"` + Debug bool `json:"debug"` + NodePrep bool `json:"nodePrep"` + Version *utils.Version `json:"version"` + HTTPRequestTimeout string `json:"httpRequestTimeout"` +} diff --git a/cli/k8s_client/yaml_factory.go b/cli/k8s_client/yaml_factory.go index dbdb1b4d2..9746081bf 100644 --- a/cli/k8s_client/yaml_factory.go +++ b/cli/k8s_client/yaml_factory.go @@ -277,22 +277,18 @@ spec: targetPort: 8001 ` -func GetCSIDeploymentYAML(deploymentName, tridentImage, - autosupportImage, autosupportProxy, autosupportCustomURL, autosupportSerialNumber, autosupportHostname, - imageRegistry, logFormat, snapshotCRDVersion string, imagePullSecrets []string, labels, - controllingCRDetails map[string]string, - debug, useIPv6, silenceAutosupport bool, version *utils.Version, topologyEnabled bool) string { +func GetCSIDeploymentYAML(args *DeploymentYAMLArguments) string { var debugLine, logLevel, ipLocalhost string - if debug { + if args.Debug { debugLine = "- -debug" logLevel = "8" } else { debugLine = "#- -debug" logLevel = "2" } - if useIPv6 { + if args.UseIPv6 { ipLocalhost = "[::1]" } else { ipLocalhost = "127.0.0.1" @@ -301,71 +297,72 @@ func GetCSIDeploymentYAML(deploymentName, tridentImage, var deploymentYAML string csiSnapshotterVersion := "v3.0.3" - switch version.MinorVersion() { + switch args.Version.MinorVersion() { case 17, 18, 19: deploymentYAML = csiDeployment117YAMLTemplate default: deploymentYAML = csiDeployment120YAMLTemplate - if snapshotCRDVersion == "v1" { + if args.SnapshotCRDVersion == "v1" { csiSnapshotterVersion = "v4.2.1" } } - if imageRegistry == "" { - imageRegistry = commonconfig.KubernetesCSISidecarRegistry + if args.ImageRegistry == "" { + args.ImageRegistry = commonconfig.KubernetesCSISidecarRegistry } - if autosupportImage == "" { - autosupportImage = commonconfig.DefaultAutosupportImage + if args.AutosupportImage == "" { + args.AutosupportImage = commonconfig.DefaultAutosupportImage } autosupportProxyLine := "" - if autosupportProxy != "" { - autosupportProxyLine = fmt.Sprint("- -proxy-url=", autosupportProxy) + if args.AutosupportProxy != "" { + autosupportProxyLine = fmt.Sprint("- -proxy-url=", args.AutosupportProxy) } autosupportCustomURLLine := "" - if autosupportCustomURL != "" { - autosupportCustomURLLine = fmt.Sprint("- -custom-url=", autosupportCustomURL) + if args.AutosupportCustomURL != "" { + autosupportCustomURLLine = fmt.Sprint("- -custom-url=", args.AutosupportCustomURL) } autosupportSerialNumberLine := "" - if autosupportSerialNumber != "" { - autosupportSerialNumberLine = fmt.Sprint("- -serial-number=", autosupportSerialNumber) + if args.AutosupportSerialNumber != "" { + autosupportSerialNumberLine = fmt.Sprint("- -serial-number=", args.AutosupportSerialNumber) } autosupportHostnameLine := "" - if autosupportHostname != "" { - autosupportHostnameLine = fmt.Sprint("- -hostname=", autosupportHostname) + if args.AutosupportHostname != "" { + autosupportHostnameLine = fmt.Sprint("- -hostname=", args.AutosupportHostname) } provisionerFeatureGates := "" - if topologyEnabled { + if args.TopologyEnabled { provisionerFeatureGates = "- --feature-gates=Topology=True" } - if labels == nil { - labels = make(map[string]string) + if args.Labels == nil { + args.Labels = make(map[string]string) } - labels[DefaultContainerLabelKey] = "trident-main" + args.Labels[DefaultContainerLabelKey] = "trident-main" - deploymentYAML = strings.ReplaceAll(deploymentYAML, "{TRIDENT_IMAGE}", tridentImage) - deploymentYAML = strings.ReplaceAll(deploymentYAML, "{DEPLOYMENT_NAME}", deploymentName) - deploymentYAML = strings.ReplaceAll(deploymentYAML, "{CSI_SIDECAR_REGISTRY}", imageRegistry) + deploymentYAML = strings.ReplaceAll(deploymentYAML, "{TRIDENT_IMAGE}", args.TridentImage) + deploymentYAML = strings.ReplaceAll(deploymentYAML, "{DEPLOYMENT_NAME}", args.DeploymentName) + deploymentYAML = strings.ReplaceAll(deploymentYAML, "{CSI_SIDECAR_REGISTRY}", args.ImageRegistry) deploymentYAML = strings.ReplaceAll(deploymentYAML, "{CSI_SNAPSHOTTER_VERSION}", csiSnapshotterVersion) deploymentYAML = strings.ReplaceAll(deploymentYAML, "{DEBUG}", debugLine) - deploymentYAML = strings.ReplaceAll(deploymentYAML, "{LABEL_APP}", labels[TridentAppLabelKey]) + deploymentYAML = strings.ReplaceAll(deploymentYAML, "{LABEL_APP}", args.Labels[TridentAppLabelKey]) deploymentYAML = strings.ReplaceAll(deploymentYAML, "{LOG_LEVEL}", logLevel) - deploymentYAML = strings.ReplaceAll(deploymentYAML, "{LOG_FORMAT}", logFormat) + deploymentYAML = strings.ReplaceAll(deploymentYAML, "{LOG_FORMAT}", args.LogFormat) deploymentYAML = strings.ReplaceAll(deploymentYAML, "{IP_LOCALHOST}", ipLocalhost) - deploymentYAML = strings.ReplaceAll(deploymentYAML, "{AUTOSUPPORT_IMAGE}", autosupportImage) + deploymentYAML = strings.ReplaceAll(deploymentYAML, "{AUTOSUPPORT_IMAGE}", args.AutosupportImage) deploymentYAML = strings.ReplaceAll(deploymentYAML, "{AUTOSUPPORT_PROXY}", autosupportProxyLine) deploymentYAML = strings.ReplaceAll(deploymentYAML, "{AUTOSUPPORT_CUSTOM_URL}", autosupportCustomURLLine) deploymentYAML = strings.ReplaceAll(deploymentYAML, "{AUTOSUPPORT_SERIAL_NUMBER}", autosupportSerialNumberLine) deploymentYAML = strings.ReplaceAll(deploymentYAML, "{AUTOSUPPORT_HOSTNAME}", autosupportHostnameLine) - deploymentYAML = strings.ReplaceAll(deploymentYAML, "{AUTOSUPPORT_SILENCE}", strconv.FormatBool(silenceAutosupport)) + deploymentYAML = strings.ReplaceAll(deploymentYAML, "{AUTOSUPPORT_SILENCE}", strconv.FormatBool(args.SilenceAutosupport)) deploymentYAML = strings.ReplaceAll(deploymentYAML, "{PROVISIONER_FEATURE_GATES}", provisionerFeatureGates) - deploymentYAML = replaceMultiline(deploymentYAML, labels, controllingCRDetails, imagePullSecrets) + deploymentYAML = strings.ReplaceAll(deploymentYAML, "{HTTP_REQUEST_TIMEOUT}", args.HTTPRequestTimeout) + deploymentYAML = replaceMultiline(deploymentYAML, args.Labels, args.ControllingCRDetails, args.ImagePullSecrets) return deploymentYAML } @@ -408,6 +405,7 @@ spec: - "--csi_role=controller" - "--log_format={LOG_FORMAT}" - "--address={IP_LOCALHOST}" + - "--http_request_timeout={HTTP_REQUEST_TIMEOUT}" - "--metrics" {DEBUG} livenessProbe: @@ -562,6 +560,7 @@ spec: - "--csi_role=controller" - "--log_format={LOG_FORMAT}" - "--address={IP_LOCALHOST}" + - "--http_request_timeout={HTTP_REQUEST_TIMEOUT}" - "--metrics" {DEBUG} livenessProbe: @@ -678,13 +677,11 @@ spec: sizeLimit: 1Gi ` -func GetCSIDaemonSetYAML(daemonsetName, tridentImage, imageRegistry, kubeletDir, logFormat, probePort string, - imagePullSecrets []string, labels, controllingCRDetails map[string]string, debug, nodePrep bool, - version *utils.Version) string { +func GetCSIDaemonSetYAML(args *DaemonsetYAMLArguments) string { var debugLine, logLevel string - if debug { + if args.Debug { debugLine = "- -debug" logLevel = "8" } else { @@ -693,31 +690,32 @@ func GetCSIDaemonSetYAML(daemonsetName, tridentImage, imageRegistry, kubeletDir, } var daemonSetYAML string - switch version.MinorVersion() { + switch args.Version.MinorVersion() { case 17: daemonSetYAML = daemonSet117YAMLTemplate default: daemonSetYAML = daemonSet118YAMLTemplate } - if imageRegistry == "" { - imageRegistry = commonconfig.KubernetesCSISidecarRegistry + if args.ImageRegistry == "" { + args.ImageRegistry = commonconfig.KubernetesCSISidecarRegistry } - labels[DefaultContainerLabelKey] = "trident-main" + args.Labels[DefaultContainerLabelKey] = "trident-main" - kubeletDir = strings.TrimRight(kubeletDir, "/") - daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{TRIDENT_IMAGE}", tridentImage) - daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DAEMONSET_NAME}", daemonsetName) - daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{CSI_SIDECAR_REGISTRY}", imageRegistry) + kubeletDir := strings.TrimRight(args.KubeletDir, "/") + daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{TRIDENT_IMAGE}", args.TridentImage) + daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DAEMONSET_NAME}", args.DaemonsetName) + daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{CSI_SIDECAR_REGISTRY}", args.ImageRegistry) daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{KUBELET_DIR}", kubeletDir) - daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{LABEL_APP}", labels[TridentAppLabelKey]) + daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{LABEL_APP}", args.Labels[TridentAppLabelKey]) daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DEBUG}", debugLine) daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{LOG_LEVEL}", logLevel) - daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{LOG_FORMAT}", logFormat) - daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{NODE_PREP}", strconv.FormatBool(nodePrep)) - daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{PROBE_PORT}", probePort) - daemonSetYAML = replaceMultiline(daemonSetYAML, labels, controllingCRDetails, imagePullSecrets) + daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{LOG_FORMAT}", args.LogFormat) + daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{NODE_PREP}", strconv.FormatBool(args.NodePrep)) + daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{PROBE_PORT}", args.ProbePort) + daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{HTTP_REQUEST_TIMEOUT}", args.HTTPRequestTimeout) + daemonSetYAML = replaceMultiline(daemonSetYAML, args.Labels, args.ControllingCRDetails, args.ImagePullSecrets) return daemonSetYAML } @@ -761,6 +759,7 @@ spec: - "--csi_role=node" - "--log_format={LOG_FORMAT}" - "--node_prep={NODE_PREP}" + - "--http_request_timeout={HTTP_REQUEST_TIMEOUT}" - "--https_rest" - "--https_port={PROBE_PORT}" {DEBUG} @@ -917,6 +916,7 @@ spec: - "--csi_role=node" - "--log_format={LOG_FORMAT}" - "--node_prep={NODE_PREP}" + - "--http_request_timeout={HTTP_REQUEST_TIMEOUT}" - "--https_rest" - "--https_port={PROBE_PORT}" {DEBUG} diff --git a/cli/k8s_client/yaml_factory_test.go b/cli/k8s_client/yaml_factory_test.go index 11a996678..cd3fb0760 100644 --- a/cli/k8s_client/yaml_factory_test.go +++ b/cli/k8s_client/yaml_factory_test.go @@ -9,6 +9,7 @@ import ( "github.com/ghodss/yaml" "github.com/stretchr/testify/assert" + "github.com/netapp/trident/config" "github.com/netapp/trident/utils" ) @@ -55,6 +56,24 @@ func TestYAMLFactory(t *testing.T) { version := utils.MustParseSemantic("1.21.0") + deploymentArgs := &DeploymentYAMLArguments{ + DeploymentName: Name, + TridentImage: ImageName, + AutosupportImage: AutosupportImageName, + AutosupportProxy: "", + AutosupportCustomURL: "", + AutosupportSerialNumber: "", + AutosupportHostname: "", + LogFormat: "", + ImageRegistry: "", + ImagePullSecrets: imagePullSecrets, + Labels: labels, + ControllingCRDetails: ownerRef, + Debug: false, + Version: version, + HTTPRequestTimeout: config.HTTPTimeoutString, + } + yamlsOutputs := []string{ GetServiceAccountYAML(Name, nil, nil, nil), GetServiceAccountYAML(Name, Secrets, labels, ownerRef), @@ -62,9 +81,7 @@ func TestYAMLFactory(t *testing.T) { GetClusterRoleYAML(FlavorOpenshift, Name, labels, ownerRef, true), GetClusterRoleBindingYAML(Namespace, FlavorOpenshift, Name, nil, ownerRef, false), GetClusterRoleBindingYAML(Namespace, FlavorK8s, Name, labels, ownerRef, true), - GetCSIDeploymentYAML(Name, ImageName, AutosupportImageName, "", "", - "", "", "", "text", "", imagePullSecrets, - labels, ownerRef, false, false, false, version, false), + GetCSIDeploymentYAML(deploymentArgs), GetCSIServiceYAML(Name, labels, ownerRef), GetSecretYAML(Name, Namespace, labels, ownerRef, nil, nil), } @@ -111,10 +128,29 @@ func TestValidateGetCSIDeploymentYAMLSuccess(t *testing.T) { version := utils.MustParseSemantic("1.17.0") + deploymentArgs := &DeploymentYAMLArguments{ + DeploymentName: "trident-csi", + TridentImage: "netapp/trident:20.10.0-custom", + AutosupportImage: "netapp/trident-autosupport:20.10.0-custom", + AutosupportSerialNumber: "0000-0000", + AutosupportHostname: "21e160d3-721f-4ec4-bcd4-c5e0d31d1a6e", + AutosupportProxy: "http://127.0.0.1/", + AutosupportCustomURL: "http://172.16.150.125:8888/", + ImageRegistry: "k8s.gcr.io", + LogFormat: "text", + SnapshotCRDVersion: "v1", + ImagePullSecrets: imagePullSecrets, + ControllingCRDetails: map[string]string{}, + Version: version, + HTTPRequestTimeout: config.HTTPTimeoutString, + TopologyEnabled: true, + Debug: true, + UseIPv6: true, + SilenceAutosupport: false, + } + yamlsOutputs := []string{ - GetCSIDeploymentYAML("trident-csi", "netapp/trident:20.10.0-custom", "netapp/trident-autosupport:20.10.0-custom", - "http://127.0.0.1/", "http://172.16.150.125:8888/", "0000-0000", "21e160d3-721f-4ec4-bcd4-c5e0d31d1a6e", - "k8s.gcr.io", "text", "v1", imagePullSecrets, labels, nil, true, true, false, version, true), + GetCSIDeploymentYAML(deploymentArgs), } for i, yamlData := range yamlsOutputs { @@ -139,10 +175,30 @@ func TestValidateGetCSIDeploymentYAMLFail(t *testing.T) { version := utils.MustParseSemantic("1.17.0") + deploymentArgs := &DeploymentYAMLArguments{ + DeploymentName: "\ntrident-csi", + TridentImage: "netapp/trident:20.10.0-custom", + AutosupportImage: "netapp/trident-autosupport:20.10.0-custom", + AutosupportProxy: "http://127.0.0.1/", + AutosupportCustomURL: "http://172.16.150.125:8888/", + AutosupportSerialNumber: "0000-0000", + AutosupportHostname: "21e160d3-721f-4ec4-bcd4-c5e0d31d1a6e", + ImageRegistry: "k8s.gcr.io", + LogFormat: "text", + SnapshotCRDVersion: "v1beta1", + ImagePullSecrets: imagePullSecrets, + Labels: labels, + ControllingCRDetails: map[string]string{}, + Debug: true, + UseIPv6: true, + SilenceAutosupport: false, + Version: version, + TopologyEnabled: true, + HTTPRequestTimeout: config.HTTPTimeoutString, + } + yamlsOutputs := []string{ - GetCSIDeploymentYAML("\ntrident-csi", "netapp/trident:20.10.0-custom", "netapp/trident-autosupport:20.10.0-custom", - "http://127.0.0.1/", "http://172.16.150.125:8888/", "0000-0000", "21e160d3-721f-4ec4-bcd4-c5e0d31d1a6e", - "k8s.gcr.io", "text", "v1beta1", imagePullSecrets, labels, nil, true, true, false, version, true), + GetCSIDeploymentYAML(deploymentArgs), } for i, yamlData := range yamlsOutputs { diff --git a/config/config.go b/config/config.go index d1d9d0769..de9e221a4 100644 --- a/config/config.go +++ b/config/config.go @@ -45,7 +45,8 @@ const ( DockerDefaultTimeout = 55 * time.Second /* REST/HTTP constants */ - HTTPTimeout = 90 * time.Second + HTTPTimeout = 90 * time.Second + HTTPTimeoutString = "90s" CACertName = "trident-ca" ServerCertName = "trident-csi" // Must match CSI service name diff --git a/frontend/rest/apiserver_http.go b/frontend/rest/apiserver_http.go index 7f6592374..1821c23c4 100644 --- a/frontend/rest/apiserver_http.go +++ b/frontend/rest/apiserver_http.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "net/http" + "time" log "github.com/sirupsen/logrus" @@ -19,7 +20,7 @@ type APIServerHTTP struct { server *http.Server } -func NewHTTPServer(p core.Orchestrator, address, port string) *APIServerHTTP { +func NewHTTPServer(p core.Orchestrator, address, port string, writeTimeout time.Duration) *APIServerHTTP { orchestrator = p @@ -28,7 +29,7 @@ func NewHTTPServer(p core.Orchestrator, address, port string) *APIServerHTTP { Addr: fmt.Sprintf("%s:%s", address, port), Handler: NewRouter(), ReadTimeout: config.HTTPTimeout, - WriteTimeout: config.HTTPTimeout, + WriteTimeout: writeTimeout, }, } diff --git a/frontend/rest/apiserver_https.go b/frontend/rest/apiserver_https.go index f1a270672..ba0128a75 100644 --- a/frontend/rest/apiserver_https.go +++ b/frontend/rest/apiserver_https.go @@ -9,6 +9,7 @@ import ( "fmt" "io/ioutil" "net/http" + "time" log "github.com/sirupsen/logrus" @@ -25,7 +26,7 @@ type APIServerHTTPS struct { func NewHTTPSServer( p core.Orchestrator, address, port, caCertFile, serverCertFile, serverKeyFile string, enableMutualTLS bool, - handler http.Handler, + handler http.Handler, writeTimeout time.Duration, ) (*APIServerHTTPS, error) { orchestrator = p @@ -36,7 +37,7 @@ func NewHTTPSServer( Handler: &tlsAuthHandler{handler: handler}, TLSConfig: &tls.Config{ClientAuth: tls.RequireAndVerifyClientCert, MinVersion: config.MinTLSVersion}, ReadTimeout: config.HTTPTimeout, - WriteTimeout: config.HTTPTimeout, + WriteTimeout: writeTimeout, }, caCertFile: caCertFile, serverCertFile: serverCertFile, diff --git a/helm/trident-operator/templates/tridentorchestrator.yaml b/helm/trident-operator/templates/tridentorchestrator.yaml index e095e9b0e..e4daa5632 100644 --- a/helm/trident-operator/templates/tridentorchestrator.yaml +++ b/helm/trident-operator/templates/tridentorchestrator.yaml @@ -7,6 +7,7 @@ spec: debug: {{ include "trident.debug" $ }} IPv6: {{ include "trident.IPv6" $ }} k8sTimeout: {{ .Values.tridentK8sTimeout }} + httpRequestTimeout: {{ .Values.tridentHttpRequestTimeout }} silenceAutosupport: {{ include "trident.silenceAutosupport" $ }} autosupportImage: {{ include "trident.autosupportImage" $ }} autosupportProxy: {{ .Values.tridentAutosupportProxy }} diff --git a/helm/trident-operator/values.yaml b/helm/trident-operator/values.yaml index 6534dee26..0592ab775 100644 --- a/helm/trident-operator/values.yaml +++ b/helm/trident-operator/values.yaml @@ -33,6 +33,10 @@ tridentIPv6: false # tridentK8sTimeout overrides the default 30-second timeout for most Kubernetes API operations (if non-zero, in seconds). tridentK8sTimeout: 0 +# tridentHttpRequestTimeout (int) overrides the default 90-second timeout for the HTTP requests, with 0 being an +# infinite duration for the timeout. A negative value will cause the default to be used. +tridentHttpRequestTimeout: -1 + # tridentSilenceAutosupport allows disabling Trident's periodic Autosupport reporting. tridentSilenceAutosupport: false diff --git a/main.go b/main.go index 4006209cc..e425eb6c7 100644 --- a/main.go +++ b/main.go @@ -70,9 +70,10 @@ var ( useCRD = flag.Bool("crd_persistence", false, "Uses CRDs for persisting orchestrator state.") // HTTP REST interface - address = flag.String("address", "127.0.0.1", "Storage orchestrator HTTP API address") - port = flag.String("port", "8000", "Storage orchestrator HTTP API port") - enableREST = flag.Bool("rest", true, "Enable HTTP REST interface") + address = flag.String("address", "127.0.0.1", "Storage orchestrator HTTP API address") + port = flag.String("port", "8000", "Storage orchestrator HTTP API port") + enableREST = flag.Bool("rest", true, "Enable HTTP REST interface") + httpRequestTimeout = flag.String("http_request_timeout", config.HTTPTimeoutString, "Set the request timeout for HTTP") // HTTPS REST interface httpsAddress = flag.String("https_address", "", "Storage orchestrator HTTPS API address") @@ -365,10 +366,19 @@ func main() { // Create HTTP REST frontend if *enableREST { + httpTimeout, err := time.ParseDuration(*httpRequestTimeout) + if err != nil { + log.Fatalf("HTTP request timeout could not be converted to a duration, cannot continue") + } + + if httpTimeout < 0 { + log.Fatalf("HTTP request timeout cannot be a negative duration, cannot continue") + } + if *port == "" { log.Warning("HTTP REST interface will not be available (port not specified).") } else { - httpServer := rest.NewHTTPServer(orchestrator, *address, *port) + httpServer := rest.NewHTTPServer(orchestrator, *address, *port, httpTimeout) preBootstrapFrontends = append(preBootstrapFrontends, httpServer) log.WithFields(log.Fields{"name": httpServer.GetName()}).Info("Added frontend.") } @@ -379,9 +389,18 @@ func main() { if *httpsPort == "" { log.Warning("HTTPS REST interface will not be available (httpsPort not specified).") } else { + httpTimeout, err := time.ParseDuration(*httpRequestTimeout) + if err != nil { + log.Fatalf("HTTP request timeout could not be converted to a duration, cannot continue") + } + + if httpTimeout < 0 { + log.Fatalf("HTTP request timeout cannot be a negative duration, cannot continue") + } + httpsServer, err := rest.NewHTTPSServer( orchestrator, *httpsAddress, *httpsPort, *httpsCACert, *httpsServerCert, *httpsServerKey, - enableMutualTLS, handler) + enableMutualTLS, handler, httpTimeout) if err != nil { log.Fatalf("Unable to start the HTTPS REST frontend. %v", err) } diff --git a/operator/controllers/orchestrator/apis/netapp/v1/types.go b/operator/controllers/orchestrator/apis/netapp/v1/types.go index e874126e4..52c4116da 100644 --- a/operator/controllers/orchestrator/apis/netapp/v1/types.go +++ b/operator/controllers/orchestrator/apis/netapp/v1/types.go @@ -35,6 +35,7 @@ type TridentOrchestratorSpec struct { Namespace string `json:"namespace"` IPv6 bool `json:"IPv6,omitempty"` K8sTimeout int `json:"k8sTimeout,omitempty"` + HTTPRequestTimeout int `json:"httpRequestTimeout,omitempty"` SilenceAutosupport bool `json:"silenceAutosupport,omitempty"` AutosupportImage string `json:"autosupportImage,omitempty"` AutosupportProxy string `json:"autosupportProxy,omitempty"` @@ -69,6 +70,7 @@ type TridentOrchestratorSpecValues struct { AutosupportSerialNumber string `json:"autosupportSerialNumber"` AutosupportHostname string `json:"autosupportHostname"` K8sTimeout string `json:"k8sTimeout"` + HTTPRequestTimeout string `json:"httpRequestTimeout"` LogFormat string `json:"logFormat"` ProbePort string `json:"probePort"` TridentImage string `json:"tridentImage"` diff --git a/operator/controllers/orchestrator/installer/installer.go b/operator/controllers/orchestrator/installer/installer.go index 92dfed7f8..c8cd73730 100644 --- a/operator/controllers/orchestrator/installer/installer.go +++ b/operator/controllers/orchestrator/installer/installer.go @@ -77,7 +77,8 @@ var ( imagePullSecrets []string - k8sTimeout time.Duration + k8sTimeout time.Duration + httpTimeout string appLabel string appLabelKey string @@ -273,6 +274,7 @@ func (i *Installer) setInstallationParams(cr netappv1.TridentOrchestrator, imageRegistry = "" kubeletDir = DefaultKubeletDir autosupportImage = commonconfig.DefaultAutosupportImage + httpTimeout = commonconfig.HTTPTimeoutString imagePullSecrets = []string{} @@ -328,6 +330,9 @@ func (i *Installer) setInstallationParams(cr netappv1.TridentOrchestrator, autosupportImage = utils.ReplaceImageRegistry(autosupportImage, cr.Spec.ImageRegistry) } } + if cr.Spec.HTTPRequestTimeout >= 0 { + httpTimeout = strconv.FormatInt(int64(cr.Spec.HTTPRequestTimeout), 10) + "s" + } appLabel = TridentCSILabel appLabelKey = TridentCSILabelKey @@ -513,6 +518,7 @@ func (i *Installer) InstallOrPatchTrident(cr netappv1.TridentOrchestrator, AutosupportHostname: autosupportHostname, KubeletDir: kubeletDir, K8sTimeout: strconv.Itoa(int(k8sTimeout.Seconds())), + HTTPRequestTimeout: httpTimeout, ImagePullSecrets: imagePullSecrets, EnableNodePrep: strconv.FormatBool(enableNodePrep), } @@ -1481,10 +1487,29 @@ func (i *Installer) createOrPatchTridentDeployment(controllingCRDetails, labels snapshotCRDVersion := i.identifyCSISnapshotterVersion(currentDeployment) - newDeploymentYAML := k8sclient.GetCSIDeploymentYAML(deploymentName, tridentImage, - autosupportImage, autosupportProxy, "", autosupportSerialNumber, autosupportHostname, - imageRegistry, logFormat, snapshotCRDVersion, imagePullSecrets, labels, controllingCRDetails, debug, useIPv6, - silenceAutosupport, i.client.ServerVersion(), topologyEnabled) + deploymentArgs := &k8sclient.DeploymentYAMLArguments{ + DeploymentName: deploymentName, + TridentImage: tridentImage, + AutosupportImage: autosupportImage, + AutosupportProxy: autosupportProxy, + AutosupportCustomURL: "", + AutosupportSerialNumber: autosupportSerialNumber, + AutosupportHostname: autosupportHostname, + ImageRegistry: imageRegistry, + LogFormat: logFormat, + SnapshotCRDVersion: snapshotCRDVersion, + ImagePullSecrets: imagePullSecrets, + Labels: labels, + ControllingCRDetails: controllingCRDetails, + Debug: debug, + UseIPv6: useIPv6, + SilenceAutosupport: silenceAutosupport, + Version: i.client.ServerVersion(), + TopologyEnabled: topologyEnabled, + HTTPRequestTimeout: httpTimeout, + } + + newDeploymentYAML := k8sclient.GetCSIDeploymentYAML(deploymentArgs) if createDeployment { // Create the deployment @@ -1603,9 +1628,22 @@ func (i *Installer) createOrPatchTridentDaemonSet(controllingCRDetails, labels m labels[appLabelKey] = TridentNodeLabelValue - newDaemonSetYAML := k8sclient.GetCSIDaemonSetYAML(daemonsetName, tridentImage, imageRegistry, kubeletDir, - logFormat, probePort, imagePullSecrets, labels, controllingCRDetails, debug, enableNodePrep, - i.client.ServerVersion()) + daemonSetArgs := &k8sclient.DaemonsetYAMLArguments{ + DaemonsetName: daemonsetName, + TridentImage: tridentImage, + ImageRegistry: imageRegistry, + KubeletDir: kubeletDir, + LogFormat: logFormat, + ProbePort: probePort, + ImagePullSecrets: imagePullSecrets, + Labels: labels, + ControllingCRDetails: controllingCRDetails, + Debug: debug, + NodePrep: enableNodePrep, + Version: i.client.ServerVersion(), + HTTPRequestTimeout: httpTimeout, + } + newDaemonSetYAML := k8sclient.GetCSIDaemonSetYAML(daemonSetArgs) if createDaemonset { // Create the daemonset