diff --git a/cmd/config_fastly.go b/cmd/config_fastly.go index af496910..fcd3879d 100644 --- a/cmd/config_fastly.go +++ b/cmd/config_fastly.go @@ -40,6 +40,10 @@ func FastlyConfigGeneration(debug bool, domain string) (lagoon.Fastly, error) { if err != nil { return lagoon.Fastly{}, fmt.Errorf("error reading fastly-service-id flag: %v", err) } + organizationVariables, err := rootCmd.PersistentFlags().GetString("organization-variables") + if err != nil { + return lagoon.Fastly{}, fmt.Errorf("error reading organization-variables flag: %v", err) + } projectVariables, err := rootCmd.PersistentFlags().GetString("project-variables") if err != nil { return lagoon.Fastly{}, fmt.Errorf("error reading project-variables flag: %v", err) @@ -53,15 +57,18 @@ func FastlyConfigGeneration(debug bool, domain string) (lagoon.Fastly, error) { fastlyServiceID = helpers.GetEnv("ROUTE_FASTLY_SERVICE_ID", fastlyServiceID, debug) // get the project and environment variables + organizationVariables = helpers.GetEnv("LAGOON_ORGANIZATION_VARIABLES", organizationVariables, debug) projectVariables = helpers.GetEnv("LAGOON_PROJECT_VARIABLES", projectVariables, debug) environmentVariables = helpers.GetEnv("LAGOON_ENVIRONMENT_VARIABLES", environmentVariables, debug) // unmarshal and then merge the two so there is only 1 set of variables to iterate over + orgVars := []lagoon.EnvironmentVariable{} projectVars := []lagoon.EnvironmentVariable{} envVars := []lagoon.EnvironmentVariable{} + json.Unmarshal([]byte(organizationVariables), &orgVars) json.Unmarshal([]byte(projectVariables), &projectVars) json.Unmarshal([]byte(environmentVariables), &envVars) - lagoonEnvVars := lagoon.MergeVariables(projectVars, envVars) + lagoonEnvVars := lagoon.MergeVariables(orgVars, projectVars, envVars, []lagoon.EnvironmentVariable{}) // generate the fastly configuration from the provided flags/variables f := &lagoon.Fastly{} diff --git a/cmd/identify_feature_test.go b/cmd/identify_feature_test.go index 7868ffe0..4f727d5f 100644 --- a/cmd/identify_feature_test.go +++ b/cmd/identify_feature_test.go @@ -158,6 +158,26 @@ func TestIdentifyFeatureFlag(t *testing.T) { }, want: "enabled", }, + { + name: "test7 check if flag is defined in lagoon organization variables", + varName: "ROOTLESS_WORKLOAD", + args: testdata.GetSeedData( + testdata.TestData{ + ProjectName: "example-project", + EnvironmentName: "main", + Branch: "main", + LagoonYAML: "internal/testdata/node/lagoon.yml", + OrganizationVariables: []lagoon.EnvironmentVariable{ + { + Name: "LAGOON_FEATURE_FLAG_ROOTLESS_WORKLOAD", + Value: "enabled", + Scope: "build", + }, + }, + }, true), + templatePath: "testoutput", + want: "enabled", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/root.go b/cmd/root.go index e88ae3cc..2f8f1096 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -124,6 +124,8 @@ func init() { "The pullrequest base branch") rootCmd.PersistentFlags().StringP("lagoon-version", "L", "", "The lagoon version") + rootCmd.PersistentFlags().StringP("organization-variables", "", "", + "The JSON payload for organization scope variables") rootCmd.PersistentFlags().StringP("project-variables", "", "", "The JSON payload for project scope variables") rootCmd.PersistentFlags().StringP("environment-variables", "", "", diff --git a/docs/buildrequirements.md b/docs/buildrequirements.md index 0b80a0bd..a2b99768 100644 --- a/docs/buildrequirements.md +++ b/docs/buildrequirements.md @@ -39,6 +39,7 @@ These are variables that are injected into a build pod by `remote-controller`, s * `PROMOTION_SOURCE_ENVIRONMENT` contains the source environment name if this is a promotion type build #### Environment Variables +* `LAGOON_ORGANIZATION_VARIABLES` contains any organization specific environment variables * `LAGOON_PROJECT_VARIABLES` contains any project specific environment variables * `LAGOON_ENVIRONMENT_VARIABLES` contains any environment specific environment variables diff --git a/internal/generator/generator.go b/internal/generator/generator.go index df8ed8b0..89b0d04b 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -33,6 +33,7 @@ type GeneratorInput struct { EnvironmentType string ActiveEnvironment string StandbyEnvironment string + OrganizationVariables string ProjectVariables string EnvironmentVariables string BuildType string @@ -139,6 +140,7 @@ func NewGenerator( buildValues.Backup.K8upVersion = helpers.GetEnv("K8UP_VERSION", generator.BackupConfiguration.K8upVersion, generator.Debug) // get the project and environment variables + organizationVariables := helpers.GetEnv("LAGOON_ORGANIZATION_VARIABLES", generator.OrganizationVariables, generator.Debug) projectVariables := helpers.GetEnv("LAGOON_PROJECT_VARIABLES", generator.ProjectVariables, generator.Debug) environmentVariables := helpers.GetEnv("LAGOON_ENVIRONMENT_VARIABLES", generator.EnvironmentVariables, generator.Debug) @@ -245,16 +247,15 @@ func NewGenerator( } // unmarshal and then merge the two so there is only 1 set of variables to iterate over + orgVars := []lagoon.EnvironmentVariable{} projectVars := []lagoon.EnvironmentVariable{} envVars := []lagoon.EnvironmentVariable{} + json.Unmarshal([]byte(organizationVariables), &orgVars) json.Unmarshal([]byte(projectVariables), &projectVars) json.Unmarshal([]byte(environmentVariables), &envVars) - mergedVariables := lagoon.MergeVariables(projectVars, envVars) // collect a bunch of the default LAGOON_X based build variables that are injected into `lagoon-env` and make them available configVars := collectBuildVariables(buildValues) - // add the calculated build runtime variables into the existing variable slice - // this will later be used to add `runtime|global` scope into the `lagoon-env` configmap - buildValues.EnvironmentVariables = lagoon.MergeVariables(mergedVariables, configVars) + buildValues.EnvironmentVariables = lagoon.MergeVariables(orgVars, projectVars, envVars, configVars) // if the core version is provided from the API, set the buildvalues LagoonVersion to this instead lagoonCoreVersion, _ := lagoon.GetLagoonVariable("LAGOON_SYSTEM_CORE_VERSION", []string{"internal_system"}, buildValues.EnvironmentVariables) diff --git a/internal/helpers/helpers.go b/internal/helpers/helpers.go index 8a01db88..38f5b1c7 100644 --- a/internal/helpers/helpers.go +++ b/internal/helpers/helpers.go @@ -218,6 +218,7 @@ func UnsetEnvVars(localVars []EnvironmentVariable) { "ACTIVE_ENVIRONMENT", "STANDBY_ENVIRONMENT", "LAGOON_FASTLY_NOCACHE_SERVICE_ID", + "LAGOON_ORGANIZATION_VARIABLES", "LAGOON_PROJECT_VARIABLES", "LAGOON_ENVIRONMENT_VARIABLES", "LAGOON_VERSION", diff --git a/internal/lagoon/variables.go b/internal/lagoon/variables.go index b3c642fa..ac070634 100644 --- a/internal/lagoon/variables.go +++ b/internal/lagoon/variables.go @@ -2,6 +2,7 @@ package lagoon import ( "fmt" + "slices" "github.com/uselagoon/build-deploy-tool/internal/helpers" ) @@ -14,44 +15,51 @@ type EnvironmentVariable struct { } // MergeVariables merges lagoon environment variables. -func MergeVariables(project, environment []EnvironmentVariable) []EnvironmentVariable { - allVars := []EnvironmentVariable{} - existsInEnvironment := false - // replace any variables from the project with ones from the environment - // this only modifies ones that exist in project - for _, pVar := range project { - add := EnvironmentVariable{} - for _, eVar := range environment { - // internal_system scoped variables are only added to the projects variabled during a build - // this make sure that any that may exist in the environment variables are not merged - // and also makes sure that internal_system variables are not replaced by other scopes - if eVar.Name == pVar.Name && pVar.Scope != "internal_system" && eVar.Scope != "internal_system" { - existsInEnvironment = true - add = eVar - } +func MergeVariables(organization, project, environment, config []EnvironmentVariable) []EnvironmentVariable { + + // Helper function to compare environment variable names. + findByName := func(name string) func(EnvironmentVariable) bool { + return func(eVar EnvironmentVariable) bool { return eVar.Name == name } + } + + // Start with config variables since they are most specific. + allVars := make([]EnvironmentVariable, len(config)) + copy(allVars, config) + + for _, eVar := range environment { + idx := slices.IndexFunc(allVars, findByName(eVar.Name)) + + // Append environment variables that are distinct. + if idx == -1 { + allVars = append(allVars, eVar) } - if existsInEnvironment { - allVars = append(allVars, add) - existsInEnvironment = false - } else { + } + + for _, pVar := range project { + idx := slices.IndexFunc(allVars, findByName(pVar.Name)) + + // Append project variables that are distinct. + if idx == -1 { allVars = append(allVars, pVar) + continue } - } - // add any that exist in the environment only to the final variables list - existsInProject := false - for _, eVar := range environment { - add := eVar - for _, aVar := range allVars { - if eVar.Name == aVar.Name { - existsInProject = true - } + + // Overwrite environment variables if they are suppossed to be internally + // scoped. + if pVar.Scope == "internal_system" { + allVars[idx] = pVar } - if existsInProject { - existsInProject = false - } else { - allVars = append(allVars, add) + } + + for _, oVar := range organization { + idx := slices.IndexFunc(allVars, findByName(oVar.Name)) + + // Append organization variables that are distinct. + if idx == -1 { + allVars = append(allVars, oVar) } } + return allVars } diff --git a/internal/lagoon/variables_test.go b/internal/lagoon/variables_test.go index 339ea24e..9bc98cb7 100644 --- a/internal/lagoon/variables_test.go +++ b/internal/lagoon/variables_test.go @@ -7,8 +7,10 @@ import ( func TestMergeVariables(t *testing.T) { type args struct { - project []EnvironmentVariable - environment []EnvironmentVariable + organization []EnvironmentVariable + project []EnvironmentVariable + environment []EnvironmentVariable + config []EnvironmentVariable } tests := []struct { name string @@ -18,6 +20,7 @@ func TestMergeVariables(t *testing.T) { { name: "test1", args: args{ + organization: []EnvironmentVariable{}, project: []EnvironmentVariable{ { Name: "PROJECT_SPECIFIC_VARIABLE", @@ -37,16 +40,17 @@ func TestMergeVariables(t *testing.T) { Scope: "global", }, }, + config: []EnvironmentVariable{}, }, want: []EnvironmentVariable{ { - Name: "PROJECT_SPECIFIC_VARIABLE", - Value: "projectvariable", + Name: "LAGOON_FASTLY_SERVICE_ID", + Value: "1234567", Scope: "global", }, { - Name: "LAGOON_FASTLY_SERVICE_ID", - Value: "1234567", + Name: "PROJECT_SPECIFIC_VARIABLE", + Value: "projectvariable", Scope: "global", }, }, @@ -54,6 +58,7 @@ func TestMergeVariables(t *testing.T) { { name: "test2", args: args{ + organization: []EnvironmentVariable{}, project: []EnvironmentVariable{ { Name: "PROJECT_SPECIFIC_VARIABLE", @@ -68,24 +73,26 @@ func TestMergeVariables(t *testing.T) { Scope: "build", }, }, + config: []EnvironmentVariable{}, }, want: []EnvironmentVariable{ - { - Name: "PROJECT_SPECIFIC_VARIABLE", - Value: "projectvariable", - Scope: "global", - }, { Name: "LAGOON_ROUTES_JSON", Value: "eyJyb3V0ZXMiOlt7ImRvbWFpbiI6InRlc3QxLmV4YW1wbGUuY29tIiwic2VydmljZSI6Im5naW54IiwidGxzLWFjbWUiOmZhbHNlLCJtb25pdG9yaW5nLXBhdGgiOiIvYnlwYXNzLWNhY2hlIn1dfQo=", Scope: "build", }, + { + Name: "PROJECT_SPECIFIC_VARIABLE", + Value: "projectvariable", + Scope: "global", + }, }, }, { name: "test3", args: args{ - project: []EnvironmentVariable{}, + organization: []EnvironmentVariable{}, + project: []EnvironmentVariable{}, environment: []EnvironmentVariable{ { Name: "LAGOON_ROUTES_JSON", @@ -93,6 +100,7 @@ func TestMergeVariables(t *testing.T) { Scope: "build", }, }, + config: []EnvironmentVariable{}, }, want: []EnvironmentVariable{ { @@ -123,13 +131,8 @@ func TestMergeVariables(t *testing.T) { Value: "123", Scope: "build", }, - { - // this will be ignored - Name: "LAGOON_ROUTE_QUOTA", - Value: "321", - Scope: "internal_system", - }, }, + config: []EnvironmentVariable{}, }, want: []EnvironmentVariable{ { @@ -137,17 +140,200 @@ func TestMergeVariables(t *testing.T) { Value: "1234", Scope: "internal_system", }, + }, + }, + { + name: "test5", + args: args{ + organization: []EnvironmentVariable{}, + project: []EnvironmentVariable{{ + Name: "LAGOON_ROUTES_JSON", + Value: "eyJyb3V0ZXMiOlt7ImRvbWFpbiI6InRlc3QxLmV4YW1wbGUuY29tIiwic2VydmljZSI6Im5naW54IiwidGxzLWFjbWUiOmZhbHNlLCJtb25pdG9yaW5nLXBhdGgiOiIvYnlwYXNzLWNhY2hlIn1dfQo=", + Scope: "build", + }}, + environment: []EnvironmentVariable{}, + config: []EnvironmentVariable{}, + }, + want: []EnvironmentVariable{ { - Name: "LAGOON_ROUTE_QUOTA", - Value: "123", + Name: "LAGOON_ROUTES_JSON", + Value: "eyJyb3V0ZXMiOlt7ImRvbWFpbiI6InRlc3QxLmV4YW1wbGUuY29tIiwic2VydmljZSI6Im5naW54IiwidGxzLWFjbWUiOmZhbHNlLCJtb25pdG9yaW5nLXBhdGgiOiIvYnlwYXNzLWNhY2hlIn1dfQo=", + Scope: "build", + }, + }, + }, + { + name: "test6", + args: args{ + organization: []EnvironmentVariable{{ + Name: "LAGOON_ROUTES_JSON", + Value: "eyJyb3V0ZXMiOlt7ImRvbWFpbiI6InRlc3QxLmV4YW1wbGUuY29tIiwic2VydmljZSI6Im5naW54IiwidGxzLWFjbWUiOmZhbHNlLCJtb25pdG9yaW5nLXBhdGgiOiIvYnlwYXNzLWNhY2hlIn1dfQo=", + Scope: "build", + }}, + project: []EnvironmentVariable{}, + environment: []EnvironmentVariable{}, + config: []EnvironmentVariable{}, + }, + want: []EnvironmentVariable{ + { + Name: "LAGOON_ROUTES_JSON", + Value: "eyJyb3V0ZXMiOlt7ImRvbWFpbiI6InRlc3QxLmV4YW1wbGUuY29tIiwic2VydmljZSI6Im5naW54IiwidGxzLWFjbWUiOmZhbHNlLCJtb25pdG9yaW5nLXBhdGgiOiIvYnlwYXNzLWNhY2hlIn1dfQo=", + Scope: "build", + }, + }, + }, + { + name: "test7", + args: args{ + organization: []EnvironmentVariable{}, + project: []EnvironmentVariable{}, + environment: []EnvironmentVariable{}, + config: []EnvironmentVariable{{ + Name: "LAGOON_ROUTES_JSON", + Value: "eyJyb3V0ZXMiOlt7ImRvbWFpbiI6InRlc3QxLmV4YW1wbGUuY29tIiwic2VydmljZSI6Im5naW54IiwidGxzLWFjbWUiOmZhbHNlLCJtb25pdG9yaW5nLXBhdGgiOiIvYnlwYXNzLWNhY2hlIn1dfQo=", Scope: "build", + }}, + }, + want: []EnvironmentVariable{ + { + Name: "LAGOON_ROUTES_JSON", + Value: "eyJyb3V0ZXMiOlt7ImRvbWFpbiI6InRlc3QxLmV4YW1wbGUuY29tIiwic2VydmljZSI6Im5naW54IiwidGxzLWFjbWUiOmZhbHNlLCJtb25pdG9yaW5nLXBhdGgiOiIvYnlwYXNzLWNhY2hlIn1dfQo=", + Scope: "build", + }, + }, + }, + { + name: "test8", + args: args{ + organization: []EnvironmentVariable{{ + Name: "ORG_KEEP", + Value: "ORG_KEEP", + Scope: "global", + }, { + Name: "OVERRIDE_1", + Value: "org", + Scope: "global", + }, { + Name: "OVERRIDE_2", + Value: "org", + Scope: "global", + }, { + Name: "OVERRIDE_3", + Value: "org", + Scope: "global", + }}, + project: []EnvironmentVariable{{ + Name: "PROJ_KEEP", + Value: "PROJ_KEEP", + Scope: "global", + }, { + Name: "OVERRIDE_1", + Value: "proj", + Scope: "global", + }, { + Name: "OVERRIDE_2", + Value: "proj", + Scope: "global", + }, { + Name: "OVERRIDE_3", + Value: "proj", + Scope: "global", + }, { + Name: "OVERRIDE_4", + Value: "proj", + Scope: "global", + }}, + environment: []EnvironmentVariable{{ + Name: "ENV_KEEP", + Value: "ENV_KEEP", + Scope: "global", + }, { + Name: "OVERRIDE_1", + Value: "env", + Scope: "global", + }, { + Name: "OVERRIDE_2", + Value: "env", + Scope: "global", + }, { + Name: "OVERRIDE_4", + Value: "env", + Scope: "global", + }, { + Name: "OVERRIDE_5", + Value: "env", + Scope: "global", + }}, + config: []EnvironmentVariable{{ + Name: "CONFIG_KEEP", + Value: "CONFIG_KEEP", + Scope: "global", + }, { + Name: "OVERRIDE_1", + Value: "config", + Scope: "global", + }, { + Name: "OVERRIDE_4", + Value: "config", + Scope: "global", + }, { + Name: "OVERRIDE_5", + Value: "config", + Scope: "global", + }}, + }, + want: []EnvironmentVariable{ + { + Name: "CONFIG_KEEP", + Value: "CONFIG_KEEP", + Scope: "global", + }, + { + Name: "OVERRIDE_1", + Value: "config", + Scope: "global", + }, + { + Name: "OVERRIDE_4", + Value: "config", + Scope: "global", + }, + { + Name: "OVERRIDE_5", + Value: "config", + Scope: "global", + }, + { + Name: "ENV_KEEP", + Value: "ENV_KEEP", + Scope: "global", + }, + { + Name: "OVERRIDE_2", + Value: "env", + Scope: "global", + }, + { + Name: "PROJ_KEEP", + Value: "PROJ_KEEP", + Scope: "global", + }, + { + Name: "OVERRIDE_3", + Value: "proj", + Scope: "global", + }, + { + Name: "ORG_KEEP", + Value: "ORG_KEEP", + Scope: "global", }, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := MergeVariables(tt.args.project, tt.args.environment); !reflect.DeepEqual(got, tt.want) { + if got := MergeVariables(tt.args.organization, tt.args.project, tt.args.environment, tt.args.config); !reflect.DeepEqual(got, tt.want) { t.Errorf("MergeVariables() = %v, want %v", got, tt.want) } }) diff --git a/internal/testdata/testdata.go b/internal/testdata/testdata.go index 06cbd2f7..62a6e8ae 100644 --- a/internal/testdata/testdata.go +++ b/internal/testdata/testdata.go @@ -41,6 +41,7 @@ type TestData struct { IngressClass string ProjectVars string EnvVars string + OrganizationVariables []lagoon.EnvironmentVariable ProjectVariables []lagoon.EnvironmentVariable EnvVariables []lagoon.EnvironmentVariable LagoonVersion string @@ -121,6 +122,11 @@ func SetupEnvironment(rootCmd cobra.Command, templatePath string, t TestData) (g if err != nil { return generator.GeneratorInput{}, err } + ov, _ := json.Marshal(t.OrganizationVariables) + err = os.Setenv("LAGOON_ORGANIZATION_VARIABLES", string(ov)) + if err != nil { + return generator.GeneratorInput{}, err + } pv, _ := json.Marshal(t.ProjectVariables) err = os.Setenv("LAGOON_PROJECT_VARIABLES", string(pv)) if err != nil { @@ -297,6 +303,9 @@ func GetSeedData(t TestData, defaultProjectVariables bool) TestData { if t.LagoonYAML != "" { rt.LagoonYAML = t.LagoonYAML } + if t.OrganizationVariables != nil { + rt.OrganizationVariables = append(rt.OrganizationVariables, t.OrganizationVariables...) + } if t.ProjectVariables != nil && defaultProjectVariables { rt.ProjectVariables = append(rt.ProjectVariables, t.ProjectVariables...) } else if !defaultProjectVariables { diff --git a/legacy/build-deploy-docker-compose.sh b/legacy/build-deploy-docker-compose.sh index 9a3f510b..defdf343 100755 --- a/legacy/build-deploy-docker-compose.sh +++ b/legacy/build-deploy-docker-compose.sh @@ -72,29 +72,15 @@ function featureFlag() { # check Lagoon project variables flagValue=$(jq -r '.[] | select(.scope == "global" and .name == "'"$flagVar"'") | .value' <<<"$LAGOON_PROJECT_VARIABLES") [ "$flagValue" ] && echo "$flagValue" && return + # check Lagoon organization variables + flagValue=$(jq -r '.[] | select(.scope == "global" and .name == "'"$flagVar"'") | .value' <<<"$LAGOON_ORGANIZATION_VARIABLES") + [ "$flagValue" ] && echo "$flagValue" && return # fall back to the default, if set. defaultFlagVar="LAGOON_FEATURE_FLAG_DEFAULT_$1" echo "${!defaultFlagVar}" } -function projectEnvironmentVariableCheck() { - # check for argument - [ "$1" ] || return - - local flagVar - - flagVar="$1" - # check Lagoon environment variables - flagValue=$(jq -r '.[] | select(.name == "'"$flagVar"'") | .value' <<<"$LAGOON_ENVIRONMENT_VARIABLES") - [ "$flagValue" ] && echo "$flagValue" && return - # check Lagoon project variables - flagValue=$(jq -r '.[] | select(.name == "'"$flagVar"'") | .value' <<<"$LAGOON_PROJECT_VARIABLES") - [ "$flagValue" ] && echo "$flagValue" && return - - echo "$2" -} - SCC_CHECK=$(kubectl -n ${NAMESPACE} get pod ${LAGOON_BUILD_NAME} -o json | jq -r '.metadata.annotations."openshift.io/scc" // false') function beginBuildStep() {